1"use strict";(self.webpackChunksemantic_router_docs=self.webpackChunksemantic_router_docs||[]).push([[24234],{77955(e,n,i){i.r(n),i.d(n,{assets:()=>a,contentTitle:()=>o,default:()=>x,frontMatter:()=>c,metadata:()=>r,toc:()=>d});const r=JSON.parse('{"id":"proposals/nvidia-dynamo-integration","title":"Semantic Intelligence Layer for NVIDIA Dynamo","description":"1. Executive Summary","source":"@site/versioned_docs/version-v0.3/proposals/nvidia-dynamo-integration.md","sourceDirName":"proposals","slug":"/proposals/nvidia-dynamo-integration","permalink":"/docs/v0.3/proposals/nvidia-dynamo-integration","draft":false,"unlisted":false,"editUrl":"https://github.com/vllm-project/semantic-router/edit/main/website/docs/proposals/nvidia-dynamo-integration.md","tags":[],"version":"v0.3","frontMatter":{},"sidebar":"tutorialSidebar","previous":{"title":"Prompt Classification Routing","permalink":"/docs/v0.3/proposals/prompt-classification-routing"},"next":{"title":"Semantic Intelligence Layer for vLLM Production\xa0Stack","permalink":"/docs/v0.3/proposals/production-stack-integration"}}');var s=i(74848),t=i(28453),l=i(2107);const c={},o="Semantic Intelligence Layer for NVIDIA Dynamo",a={},d=[{value:"1. Executive Summary",id:"1-executive-summary",level:2},{value:"2. Motivation: Why Semantic Router for Dynamo?",id:"2-motivation-why-semantic-router-for-dynamo",level:2},{value:"2.1 Dynamo Router Capabilities (Current State)",id:"21-dynamo-router-capabilities-current-state",level:3},{value:"2.2 Semantic Router Capabilities (System Intelligence Layer)",id:"22-semantic-router-capabilities-system-intelligence-layer",level:3},{value:"2.2.1 14 Domain Categories with System Prompts",id:"221-14-domain-categories-with-system-prompts",level:4},{value:"2.2.2 Fusion Routing Strategy",id:"222-fusion-routing-strategy",level:4},{value:"2.3 Differentiation Analysis: Complementary Strengths",id:"23-differentiation-analysis-complementary-strengths",level:3},{value:"Semantic Router: Request Intelligence Layer",id:"semantic-router-request-intelligence-layer",level:4},{value:"Dynamo Router: Infrastructure Efficiency Layer",id:"dynamo-router-infrastructure-efficiency-layer",level:4},{value:"Integration Value Proposition",id:"integration-value-proposition",level:4},{value:"2.4 Why Integration Matters: Achieving System-Level Intelligence",id:"24-why-integration-matters-achieving-system-level-intelligence",level:3},{value:"3. Goals and Non-Goals",id:"3-goals-and-non-goals",level:2},{value:"3.1 Goals",id:"31-goals",level:3},{value:"3.2 Non-Goals",id:"32-non-goals",level:3},{value:"4. Proposal Details",id:"4-proposal-details",level:2},{value:"4.1 Deep Learning Models",id:"41-deep-learning-models",level:3},{value:"4.1.1 Similarity Model (BERT Embeddings)",id:"411-similarity-model-bert-embeddings",level:4},{value:"4.1.2 Classification Model (Category Detection)",id:"412-classification-model-category-detection",level:4},{value:"4.1.3 PII Detection Model (Privacy Protection)",id:"413-pii-detection-model-privacy-protection",level:4},{value:"4.1.4 Jailbreak Detection Model (Security)",id:"414-jailbreak-detection-model-security",level:4},{value:"4.1.5 Model Performance Summary",id:"415-model-performance-summary",level:4},{value:"4.2 Design Principles",id:"42-design-principles",level:3},{value:"4.3 System Architecture",id:"43-system-architecture",level:3},{value:"4.4 Request Flow",id:"44-request-flow",level:3},{value:"4.4.1 End-to-End Request Processing",id:"441-end-to-end-request-processing",level:4},{value:"4.4.2 Dual-Layer Caching Strategy",id:"442-dual-layer-caching-strategy",level:4},{value:"4.5 Integration in Kubernetes",id:"45-integration-in-kubernetes",level:3},{value:"4.5.1 Deployment Architecture",id:"451-deployment-architecture",level:4},{value:"4.5.2 Service Communication Flow",id:"452-service-communication-flow",level:4},{value:"4.5.3 Worker Pool Management",id:"453-worker-pool-management",level:4},{value:"4.6 Implementation Plan",id:"46-implementation-plan",level:3},{value:"Phase 1: Foundation",id:"phase-1-foundation",level:4},{value:"Phase 2: Dual-Layer Caching",id:"phase-2-dual-layer-caching",level:4},{value:"Phase 3: Observability & Monitoring",id:"phase-3-observability--monitoring",level:4},{value:"Phase 4: Production Hardening",id:"phase-4-production-hardening",level:4},{value:"6. Security and Privacy Considerations",id:"6-security-and-privacy-considerations",level:2},{value:"6.1 PII Detection and Blocking",id:"61-pii-detection-and-blocking",level:3},{value:"6.2 Jailbreak Prevention (Prompt Guard)",id:"62-jailbreak-prevention-prompt-guard",level:3},{value:"6.3 Data Residency and Compliance",id:"63-data-residency-and-compliance",level:3},{value:"7. Operational Considerations",id:"7-operational-considerations",level:2},{value:"7.1 Monitoring and Alerting",id:"71-monitoring-and-alerting",level:3},{value:"7.3 Failure Modes and Recovery",id:"73-failure-modes-and-recovery",level:3},{value:"8. Future Enhancements",id:"8-future-enhancements",level:2},{value:"8.1 Advanced Routing Strategies",id:"81-advanced-routing-strategies",level:3},{value:"8.2 Cross-Layer Optimization",id:"82-cross-layer-optimization",level:3},{value:"8.3 Multi-Tenant Support",id:"83-multi-tenant-support",level:3},{value:"9. References",id:"9-references",level:2},{value:"9.1 NVIDIA Dynamo Documentation",id:"91-nvidia-dynamo-documentation",level:3},{value:"9.2 vLLM Semantic Router Documentation",id:"92-vllm-semantic-router-documentation",level:3},{value:"9.3 Related Research",id:"93-related-research",level:3},{value:"9.4 Integration Proposals",id:"94-integration-proposals",level:3},{value:"10. Appendix",id:"10-appendix",level:2},{value:"10.1 Glossary",id:"101-glossary",level:3},{value:"10.2 System Prompt Examples",id:"102-system-prompt-examples",level:3},{value:"10.3 API Examples",id:"103-api-examples",level:3},{value:"Conclusion",id:"conclusion",level:2}];function h(e){const n={a:"a",code:"code",h1:"h1",h2:"h2",h3:"h3",h4:"h4",header:"header",hr:"hr",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,t.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.header,{children:(0,s.jsx)(n.h1,{id:"semantic-intelligence-layer-for-nvidia-dynamo",children:"Semantic Intelligence Layer for NVIDIA Dynamo"})}),"\n",(0,s.jsx)(n.h2,{id:"1-executive-summary",children:"1. Executive Summary"}),"\n",(0,s.jsxs)(n.p,{children:["This proposal outlines a comprehensive integration strategy between ",(0,s.jsx)(n.strong,{children:"vLLM Semantic Router"})," and ",(0,s.jsx)(n.strong,{children:"NVIDIA Dynamo"}),", combining semantic intelligence with high-performance distributed inference. The integration creates a unified inference stack that leverages:"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Semantic Router's"})," intelligent request classification (14 domain categories), domain-aware system prompts, fusion rout
1ing (BERT classification + keyword matching + similarity search), security filtering, Milvus-based semantic caching"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Dynamo's"})," disaggregated serving, KV-aware routing, and multi-tier memory management"]}),"\n"]}),"\n",(0,s.jsxs)(n.p,{children:["The result is a production-grade LLM serving platform with ",(0,s.jsx)(n.strong,{children:"system-level intelligence"})," that achieves optimal balance between ",(0,s.jsx)(n.strong,{children:"accuracy"})," (routing to the right model with optimized prompts for best quality) and ",(0,s.jsx)(n.strong,{children:"efficiency"})," (maximizing GPU utilization and minimizing latency), creating a holistically intelligent inference system."]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Key Benefits:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"System-level intelligence"})," that optimally balances accuracy and efficiency across the entire inference stack"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Significant cost reduction"})," through intelligent model selection combined with infrastructure optimization"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Substantial latency improvement"})," via semantic caching + KV cache management with adaptive routing strategies"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Enhanced LLM quality"})," with domain-aware system prompts that improve Chain-of-Thought reasoning, token efficiency, and MoE expert matching"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Adaptive routing intelligence"})," with fusion routing: fast path (keyword) to deep analysis (BERT) based on query complexity, maximizing efficiency without sacrificing accuracy"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Multi-signal decision making"})," combining BERT classification, keyword matching, and similarity search for robust and accurate routing"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Holistic content safety"})," with PII detection and jailbreak prevention before inference"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"End-to-end observability"})," across semantic and infrastructure layers for continuous system optimization"]}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h2,{id:"2-motivation-why-semantic-router-for-dynamo",children:"2. Motivation: Why Semantic Router for Dynamo?"}),"\n",(0,s.jsx)(n.h3,{id:"21-dynamo-router-capabilities-current-state",children:"2.1 Dynamo Router Capabilities (Current State)"}),"\n",(0,s.jsxs)(n.p,{children:["NVIDIA Dynamo provides a sophisticated ",(0,s.jsx)(n.strong,{children:"KV-aware router"}
1)," optimized for infrastructure-level efficiency:"]}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Capability"}),(0,s.jsx)(n.th,{children:"Description"}),(0,s.jsx)(n.th,{children:"Optimization Target"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"KV Cache-Aware Routing"})}),(0,s.jsx)(n.td,{children:"Routes requests to workers with highest KV cache hit rate"}),(0,s.jsx)(n.td,{children:"TTFT, throughput"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Load-Based Routing"})}),(0,s.jsx)(n.td,{children:"Balances active decoding blocks across workers"}),(0,s.jsx)(n.td,{children:"ITL, GPU utilization"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Cost Function Optimization"})}),(0,s.jsxs)(n.td,{children:["Minimizes ",(0,s.jsx)(n.code,{children:"potential_prefill_blocks + potential_active_blocks"})]}),(0,s.jsx)(n.td,{children:"Computational cost"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Temperature-Based Selection"})}),(0,s.jsx)(n.td,{children:"Probabilistic routing to prevent worker saturation"}),(0,s.jsx)(n.td,{children:"Load distribution"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Event-Driven Tracking"})}),(0,s.jsx)(n.td,{children:"Real-time cache state via worker events"}),(0,s.jsx)(n.td,{children:"Routing accuracy"})]})]})]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Key Characteristics:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Infrastructure-focused:"})," Optimizes GPU memory and compute utilization"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Cache-aware:"})," Leverages existing KV caches to reduce prefill cost"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Load-balanced:"})," Distributes decoding workload across workers"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Performance-oriented:"})," Minimizes TTFT and ITL through smart scheduling"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"22-semantic-router-capabilities-system-intelligence-layer",children:"2.2 Semantic Router Capabilities (System Intelligence Layer)"}),"\n",(0,s.jsxs)(n.p,{children:["vLLM Semantic Router provides ",(0,s.jsx)(n.strong,{children:"system-level intelligence"})," that operates at the request understanding layer, achieving optimal balance between ",(0,s.jsx)(n.strong,{children:"accuracy"})," and ",(0,s.jsx)(n.strong,{children:"efficiency"})," through intelligent decision-making across ",(0,s.jsx)(n.strong,{children:"14 domain categories"}),":"]}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Capability"}),(0,s.jsx)(n.th,{children:"Description"}),(0,s.jsx)(n.th,{children:"Intelligence Focus"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Intent Classification"})}),(0,s.jsx)(n.td,{children:"BERT-based categorization (14 categories: math, code, business, law, etc.)"}),(0,s.jsx)(n.td,{children:"Accuracy: Precise domain understanding"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Model Selection"})}),(0,s.jsx)(n.td,{children:"Routes to best-performing model per category"}),(0,s.jsx)(n.td,{children:"Accuracy: Task-specific quality optimization"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Domain-Aware System Prompts"})}),(0,s.jsx)(n.td,{children:"Auto-injects category-specific system prompts for prompt engineering"}
1),(0,s.jsx)(n.td,{children:"Accuracy: LLM CoT quality, token efficiency, MoE expert matching"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Fusion Routing"})}),(0,s.jsx)(n.td,{children:"Multi-signal routing (keyword + similarity + BERT)"}),(0,s.jsx)(n.td,{children:"Efficiency: Adaptive latency based on query complexity"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Semantic Caching"})}),(0,s.jsx)(n.td,{children:"Milvus-based vector cache with 0.85+ similarity threshold"}),(0,s.jsx)(n.td,{children:"Efficiency: Inference cost reduction"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"PII Detection"})}),(0,s.jsx)(n.td,{children:"Token-level classification (PERSON, EMAIL, SSN, etc.)"}),(0,s.jsx)(n.td,{children:"System Intelligence: Privacy protection"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Jailbreak Prevention"})}),(0,s.jsx)(n.td,{children:"Binary classification for prompt injection attacks"}),(0,s.jsx)(n.td,{children:"System Intelligence: Security enforcement"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Tool Selection"})}),(0,s.jsx)(n.td,{children:"Semantic matching of relevant tools to reduce prompt tokens"}),(0,s.jsx)(n.td,{children:"Efficiency: Context optimization"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Reasoning Control"})}),(0,s.jsx)(n.td,{children:"Auto-enables reasoning mode for complex queries"}),(0,s.jsx)(n.td,{children:"Accuracy: Quality-aware mode selection"})]})]})]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"System Intelligence Characteristics:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Holistic Intelligence:"})," Understands query intent, complexity, and security implications across 14 domain categories"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Accuracy-Efficiency Balance:"})," Dynamically selects routing strategy (keyword/similarity/BERT) based on query complexity to maximize accuracy while minimizing latency"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Quality Optimization:"})," Selects models and prompts based on task-specific accuracy requirements"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Intelligent Prompt Engineering:"})," Auto-injects domain-specific system prompts to optimize LLM behavior and output quality"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Proactive Security:"})," Blocks malicious or privacy-violating requests before reaching inference layer"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Cost Intelligence:"})," Avoids expensive models for simple queries while ensuring quality for complex tasks"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Adaptive Routing:"})," Multi-signal fusion routing adapts to query characteristics for optimal accuracy-efficiency tradeoff"]}),"\n"]}),"\n",(0,s.jsx)(n.h4,{id:"221-14-domain-categories-with-system-prompts",children:"2.2.1 14 Domain Categories with System Prompts"}),"\n",(0,s.jsxs)(n.p,{children:["Semantic Router classifies queries into ",(0,s.jsx)(n.strong,{children:"14 specialized categories"}),": math, computer science, physics, chemistry, biology, engineering, economics, business, law, psychology, philosophy, history, health, and other. Each category has an optimized system prompt automatically injected based on query classification."]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"System Prompt Benefits:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Improved Chain-of-Thought (CoT):"})," Domain-specific prompts guide LLMs to use appropriate reasoning patterns"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:'Math: "Provide step-by-step solutions, show your work clearly"'}),"\n",(0,s.jsx)(n.li,{children:'Law: "Provide accurate legal information while clearly stating disclaimers"'}),"\n",(0,s.jsx)(n.li,{children:'Business: "Provide practical, actionable advice backed by proven methodologies"'}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Token Efficiency:"})," Optimized prompts reduce unnecessary verbosity while maintaining quality"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Shorter, focused prompts for straightforward categories (business, history)"}),"\n",(0,s.jsx)(n.li,{children:"Detailed prompts for complex domains requiring specific methodologies (math, physics)"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"MoE Expert Matching:"})," Well-crafted system prompts improve expert selection in Mixture-of-Experts models"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Domain-specific terminology activates relevant experts"}),"\n",(0,s.jsx)(n.li,{children:"Consistent prompt structure improves expert routing accuracy"}),"\n",(0,s.jsx)(n.li,{children:'Example: "You are a mathematics expert" \u2192 activates math-specialized experts in DeepSeek-V3'}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Quality Control:"})," Category-specific disclaimers and ethical guidelines"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Medical/Legal: Explicit disclaimers about professional consultation"}),"\n",(0,s.jsx)(n.li,{children:"Psychology: Emphasis on evidence-based approaches"}),"\n",(0,s.jsx)(n.li,{children:"Health: Clear boundaries between information and medical advice"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Example System Prompt (Math Category):"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"You are a mathematics expert. Provide step-by-step solutions, show your\nwork clearly, and explain mathematical concepts in an understandable way.\n"})}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Example System Prompt (Business Category):"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"You are a senior business consultant and strategic advisor with expertise\nin corporate strategy, operations management, financial analysis, marketing,\n
1and organizational development. Provide practical, actionable business advice\nbacked by proven methodologies and industry best practices. Consider market\ndynamics, competitive landscape, and stakeholder interests in your recommendations.\n"})}),"\n",(0,s.jsx)(n.h4,{id:"222-fusion-routing-strategy",children:"2.2.2 Fusion Routing Strategy"}),"\n",(0,s.jsxs)(n.p,{children:["Semantic Router implements a ",(0,s.jsx)(n.strong,{children:"multi-signal fusion routing"})," approach that combines three complementary routing methods (as detailed in the ",(0,s.jsx)(n.a,{href:"/docs/v0.3/proposals/prompt-classification-routing",children:"Prompt Classification Routing proposal"}),"):"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"1. Keyword-Based Routing (Fast Path)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:'Deterministic routing for technology-specific terms (e.g., "kubernetes", "SQL", "React")'}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Latency"}),": Minimal (significantly faster than BERT classification)"]}),"\n",(0,s.jsx)(n.li,{children:"Boolean logic support (AND/OR operators)"}),"\n",(0,s.jsx)(n.li,{children:"Easy to update without model retraining"}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Use case"}),": Exact term matching for known patterns"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"2. Similarity-Based Routing (Semantic Path)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Embedding similarity for semantic concept detection"}),"\n",(0,s.jsx)(n.li,{children:'Robust to paraphrasing ("step-by-step" \u2248 "explain thoroughly")'}),"\n",(0,s.jsx)(n.li,{children:"Configurable similarity thresholds (default: 0.75)"}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Latency"}),": Low (faster than full BERT classification)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Use case"}),": Semantic concept matching beyond exact terms"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"3. BERT Classification (Deep Understanding Path)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"14-category classification with ModernBERT"}),"\n",(0,s.jsx)(n.li,{children:"Highest accuracy for complex queries"}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Latency"}),": Moderate (comprehensive analysis)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Use case"}),": Comprehensive intent understanding"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Signal Fusion Layer:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Policy-driven decision making"}),": Combines signals with configurable priority"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Routing logic"}),":","\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsx)(n.li,{children:"Check keyword rules first (fastest)"}),"\n",(0,s.jsx)(n.li,{children:"If no keyword match, check similarity rules"}),"\n",(0,s.jsx)(n.li,{children:"If no similarity match, use BERT classification (fallback)"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Confidence scoring"}),": Each signal provides confidence score"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Override mechanism"}),": High-confidence signals can override lower-priority signals"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Observability"}),": All signals logged for analysis"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"System Intelligence Benefits of Fusion Routing:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Accuracy-Efficiency Balance"}),": Dynamically selects routing strategy based on query complexity\u2014fast path (keyword) for deterministic patterns achieves minimal latency, while deep analysis (BERT) for complex queries ensures maximum accuracy"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Adaptive Intelligence"}),": System automatically chooses the most efficient signal that meets accuracy requirements, avoiding unnecessary computation"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Flexibility"}),": Easy to add new routing rules without model retraining, enabling continuous system optimization"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Robustness"}),": Multiple signals provide redundancy and cross-validation, reducing misclassification risk and improving overall system reliability"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Holistic Optimization"}
1),": Considers both accuracy and efficiency in every routing decision, maximizing system-level intelligence"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"23-differentiation-analysis-complementary-strengths",children:"2.3 Differentiation Analysis: Complementary Strengths"}),"\n",(0,s.jsxs)(n.p,{children:["The two systems operate at ",(0,s.jsx)(n.strong,{children:"different layers"})," of the inference stack with ",(0,s.jsx)(n.strong,{children:"minimal overlap"}),":"]}),"\n",(0,s.jsx)(n.h4,{id:"semantic-router-request-intelligence-layer",children:"Semantic Router: Request Intelligence Layer"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"User Query \u2192 [Semantic Understanding] \u2192 Model Selection \u2192 Request Enrichment\n"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"What:"})," Understands query semantics, intent, and safety"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Why:"})," Routes to the right model for the task"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"When:"})," Before request reaches infrastructure"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Optimization:"})," Accuracy, cost, security"]}),"\n"]}),"\n",(0,s.jsx)(n.h4,{id:"dynamo-router-infrastructure-efficiency-layer",children:"Dynamo Router: Infrastructure Efficiency Layer"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"Enriched Request \u2192 [Worker Selection] \u2192 KV Cache Optimization \u2192 GPU Scheduling\n"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"What:"})," Optimizes worker selection and resource allocation"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Why:"})," Maximizes GPU utilization and minimizes latency"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"When:"})," After model selection, during execution"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Optimization:"})," TTFT, ITL, throughput"]}),"\n"]}),"\n",(0,s.jsx)(n.h4,{id:"integration-value-proposition",children:"Integration Value Proposition"}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Dimension"}),(0,s.jsx)(n.th,{children:"Semantic Router Alone"}),(0,s.jsx)(n.th,{children:"Dynamo Router Alone"}),(0,s.jsx)(n.th,{children:(0,s.jsx)(n.strong,{children:"Integrated System"})})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Model Selection"})}),(0,s.jsx)(n.td,{children:"\u2705 Semantic accuracy (14 categories)"}),(0,s.jsx)(n.td,{children:"\u274c No model awareness"}),(0,s.jsx)(n.td,{children:"\u2705 Best model for task"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Worker Selection"})}),(0,s.jsx)(n.td,{children:"\u274c No worker awareness"}),(0,s.jsx)(n.td,{children:"\u2705 KV cache optimization"}),(0,s.jsx)(n.td,{children:"\u2705 Optimal worker for model"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Prompt Engineering"})}),(0,s.jsx)(n.td,{children:"\u2705 Domain-aware system prompts"}),(0,s.jsx)(n.td,{children:"\u274c No prompt optimization"}),(0,s.jsx)(n.td,{children:"\u2705 Optimized CoT & MoE matching"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Fusion Routing"})}),(0,s.jsx)(n.td,{children:"\u2705 BERT + keyword + similarity fusion"}),(0,s.jsx)(n.td,{children:"\u274c KV-aware only"}),(0,s.jsx)(n.td,{children:"\u2705 Multi-signal intelligent routing"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Caching"})}),(0,s.jsx)(n.td,{children:"\u2705 Semantic similarity (Milvus)"}),(0,s.jsx)(n.td,{children:"\u2705 KV cache reuse"}),(0,s.jsxs)(n.td,{children:["\u2705\u2705 ",(0,s.jsx)(n.strong,{children:"Dual-layer caching"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Security"})}),(0,s.jsx)(n.td,{children:"\u2705 PII + jailbreak"}),(0,s.jsx)(n.td,{children:"\u274c No security layer"}),(0,s.jsx)(n.td,{children:"\u2705 Pre-inference filtering"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Cost Optimization"})}),(0,s.jsx)(n.td,{children:"\u2705 Cross-Model-level"}
1),(0,s.jsx)(n.td,{children:"\u2705 Infrastructure-level"}),(0,s.jsxs)(n.td,{children:["\u2705\u2705 ",(0,s.jsx)(n.strong,{children:"End-to-end optimization"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Latency"})}),(0,s.jsx)(n.td,{children:"Adaptive (fusion routing)"}),(0,s.jsx)(n.td,{children:"Low routing overhead"}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Parallel execution"})})]})]})]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Concrete Example:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:'Query: "Explain the proof of Fermat\'s Last Theorem step-by-step"\n\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Semantic Router Layer \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 1. Fusion Routing (3-signal analysis): \u2502\n\u2502 a) Keyword Match: "theorem", "proof" \u2192 math (confidence: 0.8)\u2502\n\u2502 b) Similarity Search: matches "mathematical proofs" concept \u2502\n\u2502 (similarity: 0.87) \u2502\n\u2502 c) BERT Classification: "math" category (confidence: 0.92) \u2502\n\u2502 \u2192 Final Decision: "math" (multi-signal consensus) \u2502\n\u2502 2. Model Selection: deepseek-v31 (best for math reasoning) \u2502\n\u2502 3. System Prompt Injection: \u2502\n\u2502 "You are a mathematics expert. Provide step-by-step \u2502\n\u2502 solutions, show your work clearly, and explain \u2502\n\u2502 mathematical concepts in an understandable way." \u2502\n\u2502 4. Reasoning Mode: ENABLED (entropy-based decision) \u2502\n\u2502 5. Security: PASS (no PII, no jailbreak) \u2502\n\u2502 6. Semantic Cache: MISS (novel query) \u2502\n\u2502 7. Enriched Request: \u2502\n\u2502 - model=deepseek-v31 \u2502\n\u2502 - reasoning_effort=high \u2502\n\u2502 - system_prompt=<math expert prompt> \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2193\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Dynamo Router Layer \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 1. Worker Pool: [worker-1, worker-2, worker-3] (deepseek-v31) \u2502\n\u2502 2. KV Cac
1he Analysis: \u2502\n\u2502 - worker-1: 15 cached blocks (math proofs context) \u2502\n\u2502 - worker-2: 3 cached blocks \u2502\n\u2502 - worker-3: 0 cached blocks \u2502\n\u2502 3. Cost Calculation: \u2502\n\u2502 - worker-1: 85 prefill + 25 active = 110 (BEST) \u2502\n\u2502 - worker-2: 97 prefill + 20 active = 117 \u2502\n\u2502 - worker-3: 100 prefill + 18 active = 118 \u2502\n\u2502 4. Selection: worker-1 (significant prefill cost reduction) \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n\nResult: \n- Right model (deepseek-v31 for math reasoning)\n- Right worker (worker-1 with relevant KV cache)\n- Right mode (reasoning enabled)\n- Significantly faster TTFT vs. random worker selection\n'})}),"\n",(0,s.jsx)(n.h3,{id:"24-why-integration-matters-achieving-system-level-intelligence",children:"2.4 Why Integration Matters: Achieving System-Level Intelligence"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Challenge 1: Infrastructure without Intelligence"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Dynamo optimizes infrastructure efficiency but lacks semantic understanding"}),"\n",(0,s.jsx)(n.li,{children:'Cannot distinguish between "2+2=?" and "Prove Fermat\'s Last Theorem"'}),"\n",(0,s.jsx)(n.li,{children:"Routes both to the same model pool without understanding complexity or quality requirements"}),"\n",(0,s.jsx)(n.li,{children:"No ability to select specialized models (math vs. code vs. creative) based on task characteristics"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Challenge 2: Intelligence without Infrastructure Awareness"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Semantic Router provides intelligent model selection but lacks infrastructure visibility"}),"\n",(0,s.jsx)(n.li,{children:"Selects the right model but not the optimal worker"}),"\n",(0,s.jsx)(n.li,{children:"Cannot leverage KV cache reuse across workers"}),"\n",(0,s.jsx)(n.li,{children:"No awareness of GPU utilization or worker load for efficiency optimization"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Solution: Holistic System Intelligence through Layered Integration"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"System Intelligence Layer (Semantic Router)\n \u2193 [accuracy: model selection, quality optimization, security]\n \u2193 [efficiency: semantic cache, adaptive routing, cost control]\nInfrastructure Optimization Layer (Dynamo)\n \u2193 [efficiency: worker selection, KV cache, GPU scheduling]\n \u2193 [accuracy: consistent execution, reliable serving]\nExecution Layer (vLLM/SGLang/TRT-LLM)\n"})}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Result:"})," A holistically intelligent system that optimizes for both accuracy (right model, right prompt, right quality) and efficiency (right worker, right cache, right resource utilization) at every layer."]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h2,{id:"3-goals-and-non-goals",children:"3. Goals and Non-Goals"}),"\n",(0,s.jsx)(n.h3,{id:"31-goals",children:"3.1 Goals"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Primary Goals:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Seamless Integration:"})," Semantic Router operates as a pre-processing layer before Dynamo's router"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Dual-Layer Caching:"})," Semantic cache (request-level) + KV cache (token-level) work in tandem"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model-Aware Routing:"})," Dynamo routes to worker pools filtered by Semantic Router's model selection"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Security Enforcement:"})," PII and jailbreak detection before requests reach Dynamo"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Unified Observability:"}
1)," Single trace spans both semantic and infrastructure layers"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Zero Downtime:"})," Hot-reload of semantic routing rules without Dynamo restart"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Secondary Goals:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Performance:"})," Combined latency < 50ms (semantic + infrastructure routing)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Scalability:"})," Support 10K+ RPS with horizontal scaling"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Flexibility:"})," Support multiple deployment patterns (sidecar, gateway, embedded)"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"32-non-goals",children:"3.2 Non-Goals"}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Replacing Dynamo Router:"})," Semantic Router augments, not replaces, Dynamo's KV-aware routing"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Modifying Dynamo Core:"})," Integration via standard APIs, no Dynamo internals changes required"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Unified Configuration:"})," Maintain separate configs for semantic and infrastructure layers"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Synchronous Coupling:"})," Systems can operate independently if needed"]}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h2,{id:"4-proposal-details",children:"4. Proposal Details"}),"\n",(0,s.jsx)(n.h3,{id:"41-deep-learning-models",children:"4.1 Deep Learning Models"}),"\n",(0,s.jsxs)(n.p,{children:["The Semantic Router leverages ",(0,s.jsx)(n.strong,{children:"four specialized deep learning models"})," for intelligent request processing. The system uses a combination of ",(0,s.jsx)(n.strong,{children:"BERT"})," and ",(0,s.jsx)(n.strong,{children:"ModernBERT"})," architectures optimized for different tasks."]}),"\n",(0,s.jsx)(n.h4,{id:"411-similarity-model-bert-embeddings",children:"4.1.1 Similarity Model (BERT Embeddings)"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Purpose:"})," Generate embeddings for semantic similarity comparison"]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Model:"})," ",(0,s.jsx)(n.code,{children:"sentence-transformers/all-MiniLM-L12-v2"})]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Key Features:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Architecture:"})," BERT-based (microsoft/MiniLM-L12-H384-uncased)","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"12 layers, 384 hidden dimensions, 12 attention heads"}),"\n",(0,s.jsx)(n.li,{children:"Fine-tuned on 1B+ sentence pairs using contrastive learning"}),"\n",(0,s.jsx)(n.li,{children:"Base model: Standard BERT architecture (not ModernBERT)"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Embedding Dimension:"})," 384"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Use Cases:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Semantic cache similarity matching (threshold: 0.8)"}),"\n",(0,s.jsx)(n.li,{children:"Tool selection via semantic search (threshold: 0.2)"}),"\n",(0,s.jsx)(n.li,{children:"Similarity-based routing for semantic concepts"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Deployment:"})," CPU-optimized for cost efficiency"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model Size:"})," 33.4M parameters (~120 MB)"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Configuration:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:"bert_model:\n model_id: sentence-transformers/all-MiniLM-L12-v2\n threshold: 0.6\n use_cpu: true\n"})}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Why BERT (not ModernBERT)?"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Mature, well-tested model with proven performance"}),"\n",(0,s.jsx)(n.li,{children:"Optimized for sentence embeddings via contrastive learning"}),"\n",(0,s.jsx)(n.li,{children:"Smaller model size (120 MB) for faster loading"}),"\n",(0,s.jsx)(n.li,{children:"ModernBERT (released Dec 2024) is used for classification tasks below"}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h4,{id:"412-classification-model-category-detection",children:"4.1.2 Classification Model (Category Detection)"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Purpose:"})," Classify queries into 14 domain categories"]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Model:"})," ",(0,s.jsx)(n.code,{children:"models/category_classifier_modernbert-base_model"})]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Key Features:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Architecture:"})," ModernBERT-base (released Dec 2024)","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Modern replacement for BERT with improved architecture"}),"\n",(0,s.jsx)(n.li,{children:"8192 token context length (vs. BERT's 512)"}),"\n",(0,s.jsx)(n.li,{children:"Rotary Position Embeddings (RoPE) for better long-context handling"}),"\n",(0,s.jsx)(n.li,{children:"Flash Attention 2 for faster inference"}),"\n",(0,s.jsx)(n.li,{children:"Fine-tuned on MMLU-Pro dataset for domain classification"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Categories:"}
1)," 14 domains (math, computer science, physics, chemistry, biology, engineering, economics, business, law, psychology, philosophy, history, health, other)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Output:"})," Category label + confidence score"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Threshold:"})," 0.6 (configurable)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Training Data:"})," MMLU-Pro dataset with domain-specific examples"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model Size:"})," ~149M parameters (ModernBERT-base)"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Configuration:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:'global:\n model_catalog:\n modules:\n classifier:\n domain:\n model_id: "models/category_classifier_modernbert-base_model"\n use_modernbert: true\n threshold: 0.6\n use_cpu: true\n category_mapping_path: "models/category_classifier_modernbert-base_model/category_mapping.json"\n'})}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Model Selection Impact:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Determines which LLM to route to (e.g., DeepSeek-V3 for math, Qwen3 for business)"}),"\n",(0,s.jsx)(n.li,{children:"Triggers domain-specific system prompt injection"}),"\n",(0,s.jsx)(n.li,{children:"Controls reasoning mode activation"}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h4,{id:"413-pii-detection-model-privacy-protection",children:"4.1.3 PII Detection Model (Privacy Protection)"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Purpose:"})," Detect personally identifiable information at token level"]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Model:"})," ",(0,s.jsx)(n.code,{children:"models/pii_classifier_modernbert-base_presidio_token_model"})]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Key Features:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Architecture:"})," ModernBERT-base fine-tuned for token classification","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Token-level sequence labeling (BIO tagging scheme)"}),"\n",(0,s.jsx)(n.li,{children:"Fine-tuned on Microsoft Presidio dataset"}),"\n",(0,s.jsx)(n.li,{children:"Optimized for privacy-sensitive entity detection"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"PII Types Detected:"})," 17 types including:","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Identity:"})," ",(0,s.jsx)(n.code,{children:"PERSON"}),", ",(0,s.jsx)(n.code,{children:"AGE"}),", ",(0,s.jsx)(n.code,{children:"NRP"})," (nationality/religious/political)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Contact:"})," ",(0,s.jsx)(n.code,{children:"EMAIL_ADDRESS"}),", ",(0,s.jsx)(n.code,{children:"PHONE_NUMBER"}),", ",(0,s.jsx)(n.code,{children:"STREET_ADDRESS"}),", ",(0,s.jsx)(n.code,{children:"ZIP_CODE"})]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Financial:"})," ",(0,s.jsx)(n.code,{children:"CREDIT_CARD"}),", ",(0,s.jsx)(n.code,{children:"IBAN_CODE"}),", ",(0,s.jsx)(n.code,{children:"US_SSN"}),", ",(0,s.jsx)(n.code,{children:"US_DRIVER_LICENSE"})]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Technical:"})," ",(0,s.jsx)(n.code,{children:"IP_ADDRESS"}),", ",(0,s.jsx)(n.code,{children:"DOMAIN_NAME"})]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Organizational:"})," ",(0,s.jsx)(n.code,{children:"ORGANIZATION"}),", ",(0,s.jsx)(n.code,{children:"GPE"})," (geopolitical entity)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Temporal:"})," ",(0,s.jsx)(n.code,{children:"DATE_TIME"})]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Granularity:"})," Token-level classification (not just entity-level)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Threshold:"})," 0.7 (configurable)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Action:"})," Block requests violating model-specific PII policies"]}
1),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model Size:"})," ~149M parameters (ModernBERT-base)"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Configuration:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:'global:\n model_catalog:\n modules:\n classifier:\n pii:\n model_id: "models/pii_classifier_modernbert-base_presidio_token_model"\n use_modernbert: true\n threshold: 0.7\n use_cpu: true\n pii_mapping_path: "models/pii_classifier_modernbert-base_presidio_token_model/pii_type_mapping.json"\n'})}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Policy Enforcement:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:'routing:\n signals:\n pii:\n - name: public-model-allow-person\n pii_types_allowed: ["PERSON"] # Only person names allowed\n decisions:\n - name: public-model-route\n rules:\n operator: AND\n conditions:\n - type: pii\n name: public-model-allow-person\n modelRefs:\n - model: public-model\n'})}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Response Headers (when blocked):"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.code,{children:"x-vsr-pii-violation: true"})}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h4,{id:"414-jailbreak-detection-model-security",children:"4.1.4 Jailbreak Detection Model (Security)"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Purpose:"})," Detect adversarial prompts and jailbreak attempts"]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Model:"})," Auto-discovered from ",(0,s.jsx)(n.code,{children:"models/"})," directory"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Key Features:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Architecture:"})," Multiple options with automatic selection","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"LoRA models (preferred):"})," Fine-tuned adapters on BERT/RoBERTa/ModernBERT base","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"lora_jailbreak_classifier_bert_model"})," (Priority 1)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"lora_jailbreak_classifier_roberta_model"})," (Priority 2)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"lora_jailbreak_classifier_modernbert_model"})," (Priority 3)"]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Legacy model (fallback):"})," ",(0,s.jsx)(n.code,{children:"jailbreak_classifier_modernbert-base_model"})]}),"\n",(0,s.jsx)(n.li,{children:"LoRA models offer better accuracy with smaller size (~10-20 MB adapters)"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model Discovery:"})," Automatic selection with architecture priority: BERT > RoBERTa > ModernBERT"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Detection Types:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Prompt injection attacks"}),"\n",(0,s.jsx)(n.li,{children:"Instruction override attempts"}),"\n",(0,s.jsx)(n.li,{children:"Adversarial prompts"}),"\n",(0,s.jsx)(n.li,{children:"Social engineering"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Threshold:"})," 0.7 (configurable)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Action:"})," Block requests with confidence above threshold"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model Size:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"LoRA: ~10-20 MB (adapter only) + base model"}),"\n",(0,s.jsx)(n.li,{children:"Legacy: ~149M parameters (ModernBERT-base)"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Configuration:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:"global:\n model_catalog:\n modules:\n prompt_guard:\n enabled: true\n use_modernbert: true\n threshold: 0.7\n use_cpu: true\n # model_id and jailbreak_mapping_path are auto-discovered\n"})}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Response Headers (when blocked):"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.code,{children:"x-vsr-jailbreak-blocked: true"})}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-jailbreak-type: {type}"}),' (e.g., "prompt_injection")']}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-jailbreak-confidence: {score}"}),' (e.g., "0.950")']}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h4,{id:"415-model-performance-summary",children:"4.1.5 Model Performance Summary"}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Model"}),(0,s.jsx)(n.th,{children:"Purpose"}),(0,s.jsx)(n.th,{children:"Architecture"}),(0,s.jsx)(n.th,{children:"Parameters"}),(0,s.jsx)(n.th,{children:"Threshold"}),(0,s.jsx)(n.th,{children:"CPU/GPU"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Similarity"})}),(0,s.jsx)(n.td,{children:"Semantic matching"}),(0,s.jsx)(n.td,{children:"BERT (MiniLM-L12)"}),(0,s.jsx)(n.td,{children:"33.4M"}),(0,s.jsx)(n.td,{children:"0.6-0.8"}),(0,s.jsx)(n.td,{children:"CPU"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Classification"})}),(0,s.jsx)(n.td,{children:"Category detection"}),(0,s.jsx)(n.td,{children:"ModernBERT-base"}),(0,s.jsx)(n.td,{children:"149M"}),(0,s.jsx)(n.td,{children:"0.6"}),(0,s.jsx)(n.td,{children:"CPU"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"PII Detection"})}),(0,s.jsx)(n.td,{children:"Privacy protection"}),(0,s.jsx)(n.td,{children:"ModernBERT-base"}),(0,s.jsx)(n.td,{children:"149M"}),(0,s.jsx)(n.td,{children:"0.7"}),(0,s.jsx)(n.td,{children:"CPU"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Jailbreak"})}),(0,s.jsx)(n.td,{children:"Security filtering"}),(0,s.jsx)(n.td,{children:"ModernBERT-base/LoRA"}),(0,s.jsx)(n.td,{children:"149M + adapters"}),(0,s.jsx)(n.td,{children:"0.7"}),(0,s.jsx)(n.td,{children:"CPU"})]})]})]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Architecture Comparison:"})}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Feature"}),(0,s.jsx)(n.th,{children:"BERT (MiniLM)"}),(0,s.jsx)(n.th,{children:"ModernBERT"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Release Date"})}),(0,s.jsx)(n.td,{children:"2020"}),(0,s.jsx)(n.td,{children:"December 2024"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Context Length"})}),(0,s.jsx)(n.td,{children:"512 tokens"}),(0,s.jsx)(n.td,{children:"8192 tokens"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Position Encoding"})}),(0,s.jsx)(n.td,{children:"Absolute"}),(0,s.jsx)(n.td,{children:"RoPE (Rotary)"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Attention"})}),(0,s.jsx)(n.td,{children:"Standard"}),(0,s.jsx)(n.td,{children:"Flash Attention 2"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Use Case"})}),(0,s.jsx)(n.td,{children:"Embeddings"}),(0,s.jsx)(n.td,{children:"Classification"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Model Size"})}),(0,s.jsx)(n.td,{children:"33.4M params"}),(0,s.jsx)(n.td,{children:"149M params"})]})]})]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Optimization Strategies:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Parallel Execution:"})," PII and Jailbreak detection run in parallel"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Early Exit:"})," Cache hits bypass all model inference"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Keyword Routing:"})," Fast path for deterministic patterns"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"CPU Optimization:"})," All models optimized for CPU inference to reduce cost"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"LoRA Adapters:"})," Jailbreak model uses lightweight adapters for faster loading"]}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h3,{id:"42-design-principles",children:"4.2 Design Principles"}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Separation of Concerns:"}
1)," Semantic intelligence and infrastructure optimization remain decoupled"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"API-Driven Integration:"})," Use Dynamo's frontend API and worker registration mechanisms"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Fail-Safe Design:"})," Semantic Router failure falls back to Dynamo's default routing"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Observability-First:"})," Every decision (semantic + infrastructure) is traced and logged"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Kubernetes-Native:"})," Designed for cloud-native deployment with CRDs and operators"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"43-system-architecture",children:"4.3 System Architecture"}),"\n","\n",(0,s.jsx)(l.A,{title:"System Architecture Overview",defaultZoom:10.5,children:'graph TB\n Client[LLM Application<br/>OpenAI SDK]\n\n subgraph Main["Main Processing Flow"]\n direction TB\n\n subgraph SIL["\u2460 vLLM Semantic Router Layer"]\n direction TB\n Gateway[Envoy Gateway :8080]\n ExtProc[Semantic Router ExtProc :50051]\n\n subgraph SC["Semantic Components"]\n direction LR\n Classifier[BERT Classifier]\n PIIDetector[PII Detector]\n JailbreakGuard[Jailbreak Guard]\n end\n\n SemanticCache[Semantic Cache]\n ToolSelector[Tool Selector]\n end\n\n subgraph DL["\u2461 NVIDIA Dynamo Layer"]\n direction TB\n DynamoFrontend[Dynamo Frontend :8000]\n\n subgraph DR["Routing & Management"]\n direction LR\n DynamoRouter[KV Router]\n KVBM[KV Block Manager]\n end\n\n Planner[Planner - Dynamic Scaling]\n end\n\n subgraph EL["\u2462 Execution Layer - Worker Pools"]\n direction TB\n\n subgraph MP1["Model Pool: deepseek-v31"]\n direction LR\n W1[Prefill Worker]\n W2[Decode Worker]\n end\n\n subgraph MP2["Model Pool: phi4"]\n direction LR\n W3[Prefill Worker]\n W4[Decode Worker]\n end\n\n subgraph MP3["Model Pool: qwen3"]\n W5[Worker - SGLang]\n end\n end\n end\n\n subgraph SL["Storage Layer"]\n direction TB\n Milvus[(Milvus<br/>Semantic Cache)]\n SystemMem[(System Memory<br/>KV Offload)]\n NVMe[(NVMe<br/>Cold Cache)]\n end\n\n Client --\x3e|1. Request| Gateway\n Gateway <--\x3e|2. ExtProc| ExtProc\n ExtProc --\x3e Classifier\n ExtProc --\x3e PIIDetector\n ExtProc --\x3e JailbreakGuard\n ExtProc --\x3e SemanticCache\n ExtProc --\x3e ToolSelector\n\n Gateway --\x3e|3. Enriched Request| DynamoFrontend\n DynamoFrontend --\x3e DynamoRouter\n DynamoRouter <--\x3e KVBM\n\n DynamoRouter --\x3e|4. Worker Selection| W1\n DynamoRouter --\x3e|4. Worker Selection| W2\n DynamoRouter -.-> W3\n DynamoRouter -.-> W4\n DynamoRouter -.-> W5\n\n Planner -.->|Scaling| W1\n Planner -.->|Scaling| W2\n Planner -.->|Scaling| W3\n Planner -.->|Scaling| W4\n Planner -.->|Scaling| W5\n\n SemanticCache <--\x3e Milvus\n KVBM <--\x3e SystemMem\n KVBM <--\x3e NVMe\n\n W1 --\x3e|5. Response| DynamoFrontend\n DynamoFrontend --\x3e|6. Response| Gateway\n Gateway --\x3e|7. Response| Client\n\n style ExtProc fill:#e1f5ff\n style DynamoRouter fill:#c8e6c9\n style SemanticCache fill:#fff9c4\n style KVBM fill:#fff9c4\n style SL fill:#f5f5f5'}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Architecture Layers:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Semantic Intelligence Layer (Semantic Router)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Envoy Gateway with ExtProc for request interception"}),"\n",(0,s.jsx)(n.li,{children:"BERT-based classification and security filtering"}),"\n",(0,s.jsx)(n.li,{children:"Semantic caching with Milvus backend"}),"\n",(0,s.jsx)(n.li,{children:"Request enrichment with routing metadata"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Infrastru
1cture Optimization Layer (Dynamo)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Dynamo Frontend receives enriched requests"}),"\n",(0,s.jsx)(n.li,{children:"KV Router performs model-aware worker selection"}),"\n",(0,s.jsx)(n.li,{children:"Planner handles dynamic scaling"}),"\n",(0,s.jsx)(n.li,{children:"KVBM manages multi-tier KV cache"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Execution Layer (vLLM/SGLang/TRT-LLM)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Model-specific worker pools"}),"\n",(0,s.jsx)(n.li,{children:"Disaggregated prefill/decode workers"}),"\n",(0,s.jsx)(n.li,{children:"Backend-agnostic execution"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Storage Layer"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Milvus for semantic cache"}),"\n",(0,s.jsx)(n.li,{children:"System memory for KV cache offload"}),"\n",(0,s.jsx)(n.li,{children:"NVMe for cold KV cache storage"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"44-request-flow",children:"4.4 Request Flow"}),"\n",(0,s.jsx)(n.h4,{id:"441-end-to-end-request-processing",children:"4.4.1 End-to-End Request Processing"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:'\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Phase 1: Semantic Intelligence (Semantic Router) \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Step 1: Request Interception \u2502\n\u2502 - Envoy Gateway receives OpenAI API request \u2502\n\u2502 - ExtProc gRPC call to Semantic Router \u2502\n\u2502 - Extract query from messages array \u2502\n\u2502 \u2502\n\u2502 Step 2: Security Filtering (Parallel Execution) \u2502\n\u2502 - PII Detection: Scan for PERSON, EMAIL, SSN, etc. \u2502\n\u2502 - Jailbreak Detection: Binary classification for prompt injection \u2502\n\u2502 - Action: BLOCK if security violation detected \u2502\n\u2502 - Latency: Low \u2502\n\u2502 \u2502\n\u2502 Step 3: Semantic Cache Lookup \u2502\n\u2502 - Generate BERT embedding for query \u2502\n\u2502 - Search Milvus for similar queries (threshold: 0.85) \u2502\n\u2502 - Action: Return cached response if HIT \u2502\n\u2502 - Latency: Very low (cache hit), Low (cache miss) \u2502\n\u2502 \u2502\n\u2502 Step 4: Fusion Routing (Multi-Signal Classification) \u2502\n\u2502 - Signal 1: Keyword matching (fast path) \u2502\n\u2502 - Signal 2: Similarity search (semantic concepts) \u2502\n\u2502 - Signal 3: BERT classification (deep understanding) \u2502\n\u2502 - Entropy-based reasoning decision \u2502\n\u2502 - Category: math, code, reasoning, creative, etc. \u2502\n\u2502 - Latency: Adaptive (keyword: minimal, similarity: low, BERT: moderate) \u2502\n\u2502 \u2502\n\u2502 Step 5: Model Selection \u2502\n\u2502 - Lookup category \u2192 model scores mapping \u2502\n\u2502 - Select best-performing model for category \u2502\n\u2502 - Example: "math" \u2192 deepseek-v31 (score: 0.92) \u2502\n\u2502 \u2502\n\u2502 Step 6: Request Enrichment \u2502\n\u2502 - Add headers: \u2502\n\u2502 * X-VSR-Model: deepseek-v31 \u2502\n\u2502 * X-VSR-Category: math \u2502\n\u2502 * X-VSR-Reasoning: true \u2502\n\u2502 * X-VSR-Reasoning-Effort: high \u2502\n\u2502 * X-VSR-Cache-Status: miss \u2502\n\u2502 - Modify request body: \u2502\n\u2502 * Update "model" field to selected model \u2502\n\u2502 * Inject reasoning parameters if applicable \u2502\n\u2502 * Add selected tools if tool selection enabled \u2502\n\u2502 \u2502\n\u2502 Total Latency: Low to Moderate (parallel execution) \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2193\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Phase 2: Infrastru
1cture Optimization (Dynamo) \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Step 7: Dynamo Frontend Receives Request \u2502\n\u2502 - Parse X-VSR-Model header \u2502\n\u2502 - Filter worker pool to model-specific workers \u2502\n\u2502 - Example: Only consider workers serving deepseek-v31 \u2502\n\u2502 \u2502\n\u2502 Step 8: KV-Aware Worker Selection \u2502\n\u2502 - Query KVBM for cached blocks per worker \u2502\n\u2502 - Calculate cost for each worker: \u2502\n\u2502 * potential_prefill_blocks = (input_tokens - overlap_blocks) / block_size\u2502\n\u2502 * potential_active_blocks = current_active + new_request_blocks \u2502\n\u2502 * logit = kv_overlap_weight \xd7 prefill + active \u2502\n\u2502 - Select worker with lowest cost \u2502\n\u2502 - Latency: Low \u2502\n\u2502 \u2502\n\u2502 Step 9: Request Forwarding \u2502\n\u2502 - Forward to selected worker (prefill or decode) \u2502\n\u2502 - Worker processes request with vLLM/SGLang/TRT-LLM \u2502\n\u2502 - KVBM tracks new KV cache blocks \u2502\n\u2502 \u2502\n\u2502 Total Latency: Low (routing overhead) \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2193\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Phase 3: Response Processing \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Step 10: Worker Response \u2502\n\u2502 - vLLM/SGLang generates tokens \u2502\n\u2502 - Stream response back to Dynamo Frontend \u2502\n\u2502 \u2502\n\u2502 Step 11: Semantic Cache Update \u2502\n\u2502 - Semantic Router receives response via ExtProc \u2502\n\u2502 - Store query embedding + response in Milvus \u2502\n\u2502 - TTL: 7200 seconds (configurable) \u2502\n\u2502 \u2502\n\u2502 Step 12: Response to Client \u2502\n\u2502 - Envoy Gateway forwards response \u2502\n\u2502 - Add response headers: \u2502\n\u2502 * X-VSR-Model-Used: deepseek-v31 \u2502\n\u2502 * X-VSR-Cache-Hit: false \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n'})}),"\n",(0,s.jsx)(n.h4,{id:"442-dual-layer-caching-strategy",children:"4.4.2 Dual-Layer C
1aching Strategy"}),"\n",(0,s.jsxs)(n.p,{children:["The integration leverages ",(0,s.jsx)(n.strong,{children:"two complementary caching layers"}),":"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Layer 1: Semantic Cache (Request-Level)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Granularity:"})," Entire request-response pairs"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Matching:"})," Embedding similarity (cosine distance)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Threshold:"})," 0.85 (configurable)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Backend:"})," Milvus (vector database)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Benefit:"})," Avoids inference entirely for similar queries"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Example:"}),' "What is 2+2?" \u2248 "Calculate 2 plus 2" (similarity: 0.91)']}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Layer 2: KV Cache (Token-Level)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Granularity:"})," Token-level KV cache blocks"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Matching:"})," Exact prefix matching"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Backend:"})," GPU HBM \u2192 System Memory \u2192 NVMe"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Benefit:"})," Reduces prefill cost for partial overlaps"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Example:"}),' "Explain quantum computing" \u2192 "Explain quantum computing applications" (prefix reuse)']}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Combined Benefit:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:'Scenario 1: Exact Semantic Match\n Query: "What is the capital of France?"\n Semantic Cache: HIT (high similarity with "What\'s France\'s capital?")\n KV Cache: N/A (inference skipped)\n Latency: Very low (cache lookup only)\n Cost Reduction: Maximum (no inference)\n\nScenario 2: Partial Semantic Match + KV Reuse\n Query: "Explain the proof of Fermat\'s Last Theorem in detail"\n Semantic Cache: MISS (novel query)\n KV Cache: HIT (significant overlap with "Explain Fermat\'s Last Theorem")\n Latency: Reduced (vs. without KV reuse)\n Cost Reduction: Significant (prefill cost saved)\n\nScenario 3: Novel Query\n Query: "Design a distributed consensus algorithm for blockchain"\n Semantic Cache: MISS\n KV Cache: MISS\n Latency: Standard (full inference)\n Cost Reduction: None (but routed to best model)\n'})}),"\n",(0,s.jsx)(n.h3,{id:"45-integration-in-kubernetes",children:"4.5 Integration in Kubernetes"}),"\n",(0,s.jsx)(n.h4,{id:"451-deployment-architecture",children:"4.5.1 Deployment Architecture"}),"\n",(0,s.jsxs)(n.p,{children:["The integration follows a ",(0,s.jsx)(n.strong,{children:"layered service architecture"})," in Kubernetes, with clear separation between semantic intelligence and infrastructure optimization:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Kubernetes Cluster: llm-inference-stack \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 \u2502\n\u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502\n\u2502 \u2502 Layer 1: Gateway & Semantic Intelligence \u2502 \u2502\n\u2502 \u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524 \u2502\n\u2502 \u2502 \u2502 \u2502\n\u2502 \u2502 [Envoy Gateway] \u2502 \u2502\n\u2502 \u2502 \u2193 (ExtProc gRP
1C) \u2502 \u2502\n\u2502 \u2502 [Semantic Router Service] \u2502 \u2502\n\u2502 \u2502 - Pods: 3 replicas (HA) \u2502 \u2502\n\u2502 \u2502 - Port: 50051 (gRPC) \u2502 \u2502\n\u2502 \u2502 - Functions: \u2502 \u2502\n\u2502 \u2502 * BERT classification (14 categories) \u2502 \u2502\n\u2502 \u2502 * System prompt injection \u2502 \u2502\n\u2502 \u2502 * PII/Jailbreak detection \u2502 \u2502\n\u2502 \u2502 * Semantic cache lookup \u2502 \u2502\n\u2502 \u2502 * Model selection \u2502 \u2502\n\u2502 \u2502 - Dependencies: \u2502 \u2502\n\u2502 \u2502 * Milvus Service (semantic cache) \u2502 \u2502\n\u2502 \u2502 * ConfigMap (routing rules) \u2502 \u2502\n\u2502 \u2502 * PVC (ML models) \u2502 \u2502\n\u2502 \u2502 \u2502 \u2502\n\u2502 \u2502 [Milvus Service] \u2502 \u2502\n\u2502 \u2502 - Port: 19530 (gRPC) \u2502 \u2502\n\u2502 \u2502 - Vector database for semantic caching \u2502 \u2502\n\u2502 \u2502 - Storage: PVC for persistence \u2502 \u2502\n\u2502 \u2502 \u2502 \u2502\n\u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502\n\u2502 \u2193 \u2502\n\u2502 (HTTP with headers: \u2502\n\u2502 X-VSR-Model, X-VSR-Category, etc.) \u2502\n\u2502 \u2193 \u2502\n\u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502\n\u2502 \u2502 Layer 2: Infrastructure Optimization (Dynamo) \u2502 \u2502\n\u2502 \u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524 \u2502\n\u2502 \u2502 \u2502 \u2502\n\u2502 \u2502 [Dynamo Frontend Service] \u2502 \u2502\n\u2502 \u2502 - Pods: 2 replicas (HA) \u2502 \u2502\n\u2502 \u2502 - Port: 8000 (HTTP) \u2502 \u2502\n\u2502 \u2502 - Functions: \u2502 \u2502\n\u2502 \u2502 * Parse X-VSR-Model header \u2502 \u2502\n\u2502 \u2502 * Filter worker pool by model \u2502 \u2502\n\u2502 \u2502 * KV-aware worker selection \u2502 \u2502\n\u2502 \u2502 * Request forwarding \u2502 \u2502\n\u2502 \u2502 - Components: \u2502 \u2502\n\u2502 \u2502 * KV Router \u2502 \u2502\n\u2502 \u2502 * Planner (dynamic scaling) \u2502 \u2502\n\u2502 \u2502 * KVBM (KV cache manager) \u2502 \u2502\n\u2502 \u2502 \u2502 \u2502\n\u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502\n\u2502 \u2193 \u2502\n\u2502 (Worker selection based on \u2502\n\u2502 model + KV cache state) \u2502\n\u2502 \u2193 \u2502\n\u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502\n\u2502 \u2502 Layer 3: Execution (vLLM/SGLang Workers) \u2502 \u2502\n\u2502 \u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524 \u2502\n\u2502 \u2502 \u2502 \u2502\n\u2502 \u2502 [Model Pool: deepseek-v31] \u2502 \u2502\n\u2502 \u2502 - StatefulSet: Multiple replicas \u2502 \u2502\n\u2502 \u2502 - Service: vllm-deepseek-v31-svc \u2502 \u2502\n\u2502 \u2502 - GPU: Multi-GPU per pod \u2502 \u2502\n\u2502 \u2502 - Features: prefix caching, fp8 KV cache \u2502 \u2502\n\u2502 \u2502 \u2502 \u2502\n\u2502 \u2502 [Model Pool: qwen3] \u2502 \u2502\n\u2502 \u2502 - StatefulSet: Multiple replicas \u2502 \u2502\n\u2502 \u2502 - Service: vllm-qwen3-svc \u2502 \u2502\n\u2502 \u2502 - GPU: Multi-GPU per pod \u2502 \u2502\n\u2502 \u2502 \u2502 \u2502\n\u2502 \u2502 [Model Pool: phi4] \u2502 \u2502\n\u2502 \u2502 - StatefulSet: Multiple replicas \u2502 \u2502\n\u2502 \u2502 - Service: vllm-phi4-svc \u2502 \u2502\n\u2502 \u2502 - GPU: Single/Multi-GPU per pod \u2502 \u2502\n\u2502 \u2502 \u2502 \u2502\n\u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502\n\u2502 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n"})}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Key Kubernetes Services:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"semantic-router-svc"})," (ClusterIP)"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Exposes Semantic Router ExtProc on port 50051"}),"\n",(0,s.jsx)(n.li,{children:"Used by Envoy Gateway for request processing"}),"\n",(0,s.jsxs)(n.li,{children:["Selector: ",(0,s.jsx)(n.code,{children:"app=semantic-router"})]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"dynamo-frontend-svc"})," (ClusterIP)"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Exposes Dynamo Frontend on port 8000"}),"\n",(0,s.jsx)(n.li,{children:"Receives enriched requests from Envoy Gateway"}),"\n",(0,s.jsxs)(n.li,{children:["Selector: ",(0,s.jsx)(n.code,{children:"app=dynamo-frontend"})]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"vllm-{model}-svc"})," (Headless Service)"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"One service per model pool"}),"\n",(0,s.jsx)(n.li,{children:"Enables direct pod-to-pod communication"}),"\n",(0,s.jsx)(n.li,{children:"Used by Dynamo for worker selection"}),"\n",(0,s.jsxs)(n.li,{children:["Selector: ",(0,s.jsx)(n.code,{children:"app=vllm-worker, model=\\{model-name\\}"})]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"milvus-svc"})," (ClusterIP)"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Exposes Milvus on port 19530 (gRPC)"}),"\n",(0,s.jsx)(n.li,{children:"Used by Semantic Router for semantic caching"}),"\n",(0,s.jsx)(n.li,{children:"Vector database for embedding similarity search"}),"\n",(0,s.jsxs)(n.li,{children:["Selector: ",(0,s.jsx)(n.code,{children:"app=milvus"})]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h4,{id:"452-service-communication-flow",children:"4.5.2 Service Communication Flow"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"End-to-End Request Path:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:'\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Step 1: Client Request \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 POST /v1/chat/completions \u2502\n\u2502 Host: llm-gateway.example.com:8080 \u2502\n\u2502 Content-Type: application/json \u2502\n\u2502 \u2502\n\u2502 { \u2502\n\u2502 "messages": [ \u2502\n\u2502 {"role": "user", "content": "Prove Fermat\'s Last Theorem"} \u2502\n\u2502 ], \u2502\n\u2502 "model": "auto" \u2502\n\u2502 } \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2193\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Step 2: Envoy Gateway (Port 8080) \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 - Receives HTTP request \u2502\n\u2502 - Invokes ExtProc: semantic-router-svc:
150051 (gRPC) \u2502\n\u2502 - Sends request body + headers to Semantic Router \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2193\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Step 3: Semantic Router Service (ExtProc gRPC) \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Processing Pipeline: \u2502\n\u2502 \u2502\n\u2502 3.1 Fusion Routing (Multi-Signal Classification) \u2502\n\u2502 - Input: "Prove Fermat\'s Last Theorem" \u2502\n\u2502 - Keyword matching: No match \u2502\n\u2502 - Similarity search: No strong match \u2502\n\u2502 - BERT classification: category="math", confidence=0.92 \u2502\n\u2502 - Decision: Use BERT result (highest confidence) \u2502\n\u2502 \u2502\n\u2502 3.2 System Prompt Selection \u2502\n\u2502 - Lookup: categories["math"].system_prompt \u2502\n\u2502 - Prompt: "You are a mathematics expert..." \u2502\n\u2502 \u2502\n\u2502 3.3 Model Selection \u2502\n\u2502 - Lookup: categories["math"].model_scores \u2502\n\u2502 - Selected: deepseek-v31 (score: 0.92, reasoning: true) \u2502\n\u2502 \u2502\n\u2502 3.4 Security Checks \u2502\n\u2502 - PII Detection: PASS (no sensitive data) \u2502\n\u2502 - Jailbreak Detection: PASS (legitimate query) \u2502\n\u2502 \u2502\n\u2502 3.5 Semantic Cache Lookup \u2502\n\u2502 - Query Milvus: embedding similarity search \u2502\n\u2502 - Result: MISS (novel query) \u2502\n\u2502 \u2502\n\u2502 3.6 Response to Envoy \u2502\n\u2502 - Modified Request Body: \u2502\n\u2502 * model: "auto" \u2192 "deepseek-v31" (OVERRIDDEN) \u2502\n\u2502 * messages: [system prompt injected] \u2502\n\u2502 - Observability Headers (optional, added to response): \u2502\n\u2502 * x-vsr-selected-category: math \u2502\n\u2502 * x-vsr-selected-reasoning: on \u2502\n\u2502 * x-vsr-selected-model: deepseek-v31 \u2502\n\u2502 * x-vsr-injected-system-prompt: true \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2193\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Step 4: Envoy Gateway (Forwarding) \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 - Receives enriched request from Semantic Router \u2502\n\u2502 - Forwards to: dynamo-frontend-svc:8000 \u2502\n\u2502 - Request body now has: model="deepseek-v31" (overridden from "auto")\u2502\n\u2502 - Optional observability headers preserved \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2193\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Step 5: Dynamo Frontend Service (Port 8000) \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Processing Pipeline: \u2502\n\u2502 \u2502\n\u2502 5.1 Request Body Parsing \u2502\n\u2502 - Read: request.model = "deepseek-v31" \u2502\n\u2502 - Dynamo is UNAWARE that model was changed by VSR \u2502\n\u2502 - Treats it as a normal request for deepseek-v31 \u2502\n\u2502 \u2502\n\u2502 5.2 Worker Pool Filtering \u2502\n\u2502 - Query Kubernetes: vllm-deepseek-v31-svc (Headless) \u2502\n\u2502 - Available Workers: \u2502\n\u2502 * vllm-deepseek-v31-0 (10.244.1.5:8000) \u2502\n\u2502 * vllm-deepseek-v31-1 (10.244.1.6:8000) \u2502\n\u2502 * vllm-deepseek-v31-2 (10.244.1.7:8000) \u2502\n\u2502 * vllm-deepseek-v31-3 (10.244.1.8:8000) \u2502\n\u2502 \u2502\n\u2502 5.3 KV-Aware Worker Selection \u2502\n\u2502 - Query KVBM for each worker\'s cac
1he state \u2502\n\u2502 - Calculate routing score: \u2502\n\u2502 score = kv_overlap \xd7 weight + active_blocks \u2502\n\u2502 - Results: \u2502\n\u2502 * Worker-0: score=120 (high KV overlap) \u2502\n\u2502 * Worker-1: score=85 \u2502\n\u2502 * Worker-2: score=90 \u2502\n\u2502 * Worker-3: score=75 \u2502\n\u2502 - Selected: Worker-0 (10.244.1.5:8000) \u2502\n\u2502 \u2502\n\u2502 5.4 Request Forwarding \u2502\n\u2502 - Forward to: http://10.244.1.5:8000/v1/chat/completions \u2502\n\u2502 - Request body: model="deepseek-v31" (as-is from VSR) \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2193\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Step 6: vLLM Worker (deepseek-v31-0) \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 6.1 Request Processing \u2502\n\u2502 - Receive request: model="deepseek-v31" \u2502\n\u2502 - System prompt already injected in messages by VSR \u2502\n\u2502 - Worker is UNAWARE of VSR\'s involvement \u2502\n\u2502 \u2502\n\u2502 6.2 Inference Execution \u2502\n\u2502 - Model: DeepSeek-V3 \u2502\n\u2502 - Messages: [system prompt + user query] \u2502\n\u2502 - Prefix Caching: Enabled (KV cache reuse) \u2502\n\u2502 - Generate response with step-by-step proof \u2502\n\u2502 \u2502\n\u2502 6.3 Response Generation \u2502\n\u2502 - Return: Streaming or non-streaming response \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2193\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Step 7: Response Path (Reverse) \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Worker \u2192 Dynamo Frontend \u2192 Envoy Gateway \u2192 Client \u2502\n\u2502 \u2502\n\u2502 - Envoy adds observability headers: X-Envoy-Upstream-Service-Time \u2502\n\u2502 - Client receives complete response with metadata \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n'})}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Key Integration Points:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Transparent Model Override (Critical Design)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["User sends: ",(0,s.jsx)(n.code,{children:'{"model": "auto", "messages": [...]}'})]}),"\n",(0,s.jsxs)(n.li,{children:["Semantic Router modifies request body: ",(0,s.jsx)(n.code,{children:'model: "auto" \u2192 "deepseek-v31"'})]}),"\n",(0,s.jsxs)(n.li,{children:["Dynamo receives: ",(0,s.jsx)(n.code,{children:'{"model": "deepseek-v31", "messages": [...]}'})]}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.strong,{children:"Dynamo is completely unaware of VSR's involvement"})}),"\n",(0,s.jsx)(n.li,{children:"No special headers needed for model routing"}),"\n",(0,s.jsx)(n.li,{children:"Standard OpenAI API compatibility maintained"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"System Prompt Injection"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Semantic Router injects system prompt into messages array"}),"\n",(0,s.jsxs)(n.li,{children:["Example: ",(0,s.jsx)(n.code,{children:'messages: [{"role": "system", "content": "You are a mathematics expert..."}, {"role": "user", "content": "..."}]'})]}),"\n",(0,s.jsx)(n.li,{children:"Worker receives pre-enriched request"}),"\n",(0,s.jsx)(n.li,{children:"No additional processing needed by Dynamo or worker"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Service Discovery"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["Envoy \u2192 Semantic Router: ",(0,s.jsx)(n.code,{children:"semantic-router-svc.llm-inference-stack.svc.cluster.local:50051"})," (gRPC ExtProc)"]}),"\n",(0,s.jsxs)(n.li,{children:["Envoy \u2192 Dynamo: ",(0,s.jsx)(n.code,{children:"dynamo-frontend-svc.llm-inference-stack.svc.cluster.local:8000"})," (HTTP)"]}),"\n",(0,s.jsxs)(n.li,{children:["Dynamo \u2192 Workers: ",(0,s.jsx)(n.code,{children:"vllm-\\{model\\}-svc.llm-inference-stack.svc.cluster.local"})," (Headless Service)"]}),"\n",(0,s.jsxs)(n.li,{children:["Semantic Router \u2192 Milvus: ",(0,s.jsx)(n.code,{children:"milvus-svc.llm-inference-stack.svc.cluster.local:19530"})," (gRPC)"]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Observability (Optional Headers)"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-selected-category"}),': Query classification result (e.g., "math")']}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-selected-reasoning"}),': Reasoning mode flag (e.g., "on" or "off")']}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-selected-model"}),': Model selected by VSR (e.g., "deepseek-v31")']}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-injected-system-prompt"}),': Whether system prompt was injected (e.g., "true" or "false")']}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-cache-hit"}),': Semantic cache status (value: "true" when cache hit)']}),"\n",(0,s.jsxs)(n.li,{children:["These headers are for ",(0,s.jsx)(n.strong,{children:"observability only"}),", not used by Dynamo for routing"]}),"\n",(0,s.jsx)(n.li,{children:"Dynamo and workers can ignore these headers"}),"\n",(0,s.jsx)(n.li,{children:"Headers are only added to successful responses (HTTP 200-299) that did not hit cache"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Distributed Tracing"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Full-stack distributed tracing support across VSR \u2192 Dynamo \u2192 Workers"}),"\n",(0,s.jsx)(n.li,{children:"OpenTelemetry-based instrumentation"}),"\n",(0,s.jsx)(n.li,{children:"Single trace spans all layers with proper context propagation"}),"\n",(0,s.jsxs)(n.li,{children:["Reference: ",(0,s.jsx)(n.a,{href:"https://github.com/vllm-project/semantic-router/pull/322",children:"PR #322 - Distributed Tracing Support"})]}
1),"\n",(0,s.jsx)(n.li,{children:"Enables end-to-end latency analysis and bottleneck identification"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Cache Coordination"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Semantic cache (Milvus): Request-level, checked first by VSR"}),"\n",(0,s.jsx)(n.li,{children:"KV cache (Dynamo/vLLM): Token-level, managed by Dynamo"}),"\n",(0,s.jsx)(n.li,{children:"Independent layers, no coordination needed"}),"\n",(0,s.jsx)(n.li,{children:"If semantic cache hits, request never reaches Dynamo"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h4,{id:"453-worker-pool-management",children:"4.5.3 Worker Pool Management"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Worker Discovery via Kubernetes Services:"})}),"\n",(0,s.jsx)(n.p,{children:"Dynamo Frontend discovers workers through Kubernetes Headless Services, which provide direct pod IP addresses:"}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Headless Service Configuration"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["Service Type: ",(0,s.jsx)(n.code,{children:"ClusterIP: None"})," (headless)"]}),"\n",(0,s.jsxs)(n.li,{children:["Selector: ",(0,s.jsx)(n.code,{children:"app=vllm-worker, model=\\{model-name\\}"})]}),"\n",(0,s.jsx)(n.li,{children:"DNS returns all pod IPs instead of load-balanced VIP"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Worker Registration Flow"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"vLLM Worker Pod Startup\n\u2193\nWorker registers with Dynamo Frontend via HTTP API\n\u2193\nDynamo Frontend tracks:\n- Worker ID (pod name)\n- Model name (deepseek-v31, qwen3, phi4)\n- Endpoint (pod IP:8000)\n- Capabilities (prefill, decode, max_batch_size)\n- KV cache state (tracked by KVBM)\n"})}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Model Pool Organization"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Each model has dedicated StatefulSet + Headless Service"}),"\n",(0,s.jsxs)(n.li,{children:["Example: ",(0,s.jsx)(n.code,{children:"vllm-deepseek-v31-svc"})," \u2192 4 pods serving DeepSeek-V3"]}),"\n",(0,s.jsx)(n.li,{children:"Dynamo queries service DNS to get all pod IPs"}),"\n",(0,s.jsxs)(n.li,{children:["Filters workers by ",(0,s.jsx)(n.code,{children:"X-VSR-Model"})," header from Semantic Router"]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Dynamic Scaling"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Horizontal Pod Autoscaler (HPA) adjusts replicas based on GPU utilization"}),"\n",(0,s.jsx)(n.li,{children:"New pods auto-register with Dynamo on startup"}),"\n",(0,s.jsx)(n.li,{children:"Dynamo updates worker pool in real-time"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"46-implementation-plan",children:"4.6 Implementation Plan"}),"\n",(0,s.jsx)(n.h4,{id:"phase-1-foundation",children:"Phase 1: Foundation"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Objectives:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Establish basic integration between Semantic Router and Dynamo"}),"\n",(0,s.jsx)(n.li,{children:"Implement transparent model override in request body"}),"\n",(0,s.jsx)(n.li,{children:"Validate end-to-end request flow"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Tasks:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Semantic Router Enhancements:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["Implement request body modification: ",(0,s.jsx)(n.code,{children:'model: "auto" \u2192 "selected-model"'})]}),"\n",(0,s.jsx)(n.li,{children:"Add system prompt injection to messages array"}),"\n",(0,s.jsxs)(n.li,{children:["Add optional observability headers:","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-selected-category"}),": Classification result"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-selected-reasoning"}),': Reasoning mode ("on" or "off")']}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-selected-model"}),": Selected model name"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-injected-system-prompt"}),': System prompt injection status ("true" or "false")']}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-cache-hit"}),": Cache hit status (only when cache hit)"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.li,{children:"Ensure OpenAI API compatibility maintained"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Dynamo Frontend (No Changes Required):"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Dynamo receives standard OpenAI API requests"}),"\n",(0,s.jsx)(n.li,{children:"Model field already contains the selected model name"}),"\n",(0,s.jsx)(n.li,{children:"No awareness of VSR's involvement needed"}),"\n",(0,s.jsx)(n.li,{children:"Existing routing logic works as-is"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Testing:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Unit tests for model override logic"}),"\n",(0,s.jsx)(n.li,{children:"Integration tests for system prompt injection"}),"\n",(0,s.jsx)(n.li,{children:"Verify Dynamo routes to correct model pools"}),"\n",(0,s.jsx)(n.li,{children:"Load tests with 1K RPS"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Success Criteria:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"\u2705 Requests routed to correct model pools based on overridden model name"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 System prompts correctly injected into messages"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 Dynamo operates transparently without modifications"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 Latency overhead < 10ms"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 No breaking changes to existing deployments"}),"\n"]}),"\n",(0,s.jsx)(n.h4,{id:"phase-2-dual-layer-caching",children:"Phase 2: Dual-Layer C
1aching"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Objectives:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Integrate semantic cache with KV cache"}),"\n",(0,s.jsx)(n.li,{children:"Implement cache coordination strategy"}),"\n",(0,s.jsx)(n.li,{children:"Optimize cache hit rates"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Tasks:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Cache Integration:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Add semantic cache lookup before Dynamo routing"}),"\n",(0,s.jsx)(n.li,{children:"Implement cache miss forwarding to Dynamo"}),"\n",(0,s.jsx)(n.li,{children:"Add cache hit metrics and headers"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Performance Optimization:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Parallel cache lookup and classification"}),"\n",(0,s.jsx)(n.li,{children:"Milvus connection pooling"}),"\n",(0,s.jsx)(n.li,{children:"Cache warming strategies"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Testing:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Cache hit rate benchmarks"}),"\n",(0,s.jsx)(n.li,{children:"Latency comparison (cache hit vs. miss)"}),"\n",(0,s.jsx)(n.li,{children:"Cache eviction policy validation"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Success Criteria:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"\u2705 High semantic cache hit rate (production workloads)"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 Low cache hit latency"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 High combined cache hit rate (semantic + KV)"}),"\n"]}),"\n",(0,s.jsx)(n.h4,{id:"phase-3-observability--monitoring",children:"Phase 3: Observability & Monitoring"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Objectives:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Full-stack distributed tracing across VSR \u2192 Dynamo \u2192 Workers"}),"\n",(0,s.jsx)(n.li,{children:"Comprehensive metrics and dashboards"}),"\n",(0,s.jsx)(n.li,{children:"Alerting and SLO monitoring"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Tasks:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Distributed Tracing (OpenTelemetry):"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Trace context propagation from VSR through Dynamo to workers"}),"\n",(0,s.jsxs)(n.li,{children:["Span hierarchy:","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Root span: Envoy Gateway"}),"\n",(0,s.jsxs)(n.li,{children:["Child span: Semantic Router (fusion routing, cache, security)","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Sub-span: BERT classification"}),"\n",(0,s.jsx)(n.li,{children:"Sub-span: Keyword matching"}),"\n",(0,s.jsx)(n.li,{children:"Sub-span: Similarity search"}),"\n",(0,s.jsx)(n.li,{children:"Sub-span: Signal fusion & decision"}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.li,{children:"Child span: Dynamo Frontend (routing, worker selection)"}),"\n",(0,s.jsx)(n.li,{children:"Child span: vLLM Worker (inference execution)"}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.li,{children:"Automatic trace ID injection in headers"}),"\n",(0,s.jsx)(n.li,{children:"Support for Jaeger, Tempo, and other OTLP-compatible backends"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Metrics Collection:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["Semantic Router metrics:","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["Fusion routing performance:","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"BERT classification latency and accuracy"}),"\n",(0,s.jsx)(n.li,{children:"Keyword matching hit rate and latency"}),"\n",(0,s.jsx)(n.li,{children:"Similarity search latency"}),"\n",(0,s.jsx)(n.li,{children:"Signal fusion decision distribution"}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.li,{children:"Semantic cache hit rate (Milvus)"}),"\n",(0,s.jsx)(n.li,{children:"PII/Jailbreak detection rate"}),"\n",(0,s.jsx)(n.li,{children:"Model selection distribution by category"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["Dynamo metrics:","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"KV-aware routing decisions"}),"\n",(0,s.jsx)(n.li,{children:"Worker utilization"}),"\n",(0,s.jsx)(n.li,{children:"KV cache hit rate"}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.li,{children:"End-to-end latency breakdown by component"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Dashboards:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Grafana dashboard for integrated stack"}),"\n",(0,s.jsx)(n.li,{children:"Request flow visualization with trace waterfall"}),"\n",(0,s.jsx)(n.li,{children:"Cost and performance analytics"}),"\n",(0,s.jsx)(n.li,{children:"Cache efficiency metrics (semantic + KV)"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Success Criteria:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"\u2705 Single distributed trace spans all layers (VSR \u2192 Dynamo \u2192 Worker)"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 Minimal trace sampling overhead"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 Real-time dashboards operational"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 Trace context properly propagated across service boundaries"}),"\n"]}),"\n",(0,s.jsx)(n.h4,{id:"phase-4-production-hardening",children:"Phase 4: Production Hardening"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Objectives:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Failure handling and resilience"}),"\n",(0,s.jsx)(n.li,{children:"Performance optimization"}),"\n",(0,s.jsx)(n.li,{children:"Production deployment"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Tasks:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Resilience:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Semantic Router failure fallback to Dynamo"}),"\n",(0,s.jsx)(n.li,{children:"Circuit breaker for cache backend"}),"\n",(0,s.jsx)(n.li,{children:"Graceful degradation strategies"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Performance:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Latency optimization (target: < 50ms combined)"}),"\n",(0,s.jsx)(n.li,{children:"Throughput testing (target: 10K RPS)"}),"\n",(0,s.jsx)(n.li,{children:"Resource utilization tuning"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Documentation:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Deployment guide"}),"\n",(0,s.jsx)(n.li,{children:"Configuration reference"}),"\n",(0,s.jsx)(n.li,{children:"Troubleshooting runbook"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Success Criteria:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"\u2705 High availability"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 Low P99 latency (routing overhead)"}),"\n",(0,s.jsx)(n.li,{children:"\u2705 10K+ RPS sustained throughput"}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h2,{id:"6-security-and-privacy-considerations",children:"6. Security and Privacy Considerations"}),"\n",(0,s.jsx)(n.h3,{id:"61-pii-detection-and-blocking",children:"6.1 PII Detection and Blocking"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Threat Model:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Users may inadvertently include PII in prompts"}),"\n",(0,s.jsx)(n.li,{children:"PII could be logged, cached, or sent to third-party models"}),"\n",(0,s.jsx)(n.li,{children:"Compliance requirements (GDPR, HIPAA, CCPA)"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Mitigation:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Token-level PII detection using ModernBERT classifier"}),"\n",(0,s.jsx)(n.li,{children:"Configurable blocking policies per model"}),"\n",(0,s.jsx)(n.li,{children:"PII types: PERSON, EMAIL_ADDRESS, PHONE_NUMBER, US_SSN, CREDIT_CARD, STREET_ADDRESS, IP_ADDRESS, IBAN_CODE, US_DRIVER_LICENSE, and more"}),"\n",(0,s.jsxs)(n.li,{children:["Response header when blocked: ",(0,s.jsx)(n.code,{children:"x-vsr-pii-violation: true"})]}),"\n",(0,s.jsx)(n.li,{children:"Audit logging of all PII detections"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Example Configuration:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:'routing:\n signals:\n pii:\n - name: public-model-allow-person\n pii_types_allowed: ["PERSON"] # Only person names allowed\n decisions:\n - name: public-model-route\n rules:\n operator: AND\n conditions:\n - type: pii\n name: public-model-allow-person\n modelRefs:\n - model: public-model\n'})}),"\n",(0,s.jsx)(n.h3,{id:"62-jailbreak-prevention-prompt-guard",children:"6.2 Jailbreak Prevention (Prompt Guard)"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Threat Model:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Adversarial prompts attempting to bypass safety guardrails"}),"\n",(0,s.jsx)(n.li,{children:"Prompt injection attacks"}),"\n",(0,s.jsx)(n.li,{children:"Social engineering attempts"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Mitigation:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Prompt Guard"})," classification for jailbreak detection"]}),"\n",(0,s.jsx)(n.li,{children:"Threshold-based blocking (configurable, e.g., 0.5)"}),"\n",(0,s.jsx)(n.li,{children:"ModernBERT-based classification model"}),"\n",(0,s.jsx)(n.li,{children:"Jailbreak type detection with confidence scoring"}),"\n",(0,s.jsxs)(n.li,{children:["Response headers when blocked:","\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.code,{children:"x-vsr-jailbreak-blocked: true"})}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-jailbreak-type: {type}"}),' (e.g., "prompt_injection")']}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"x-vsr-jailbreak-confidence: {score}"}),' (e.g., "0.950")']}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Example Configuration:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:"global:\n model_catalog:\n modules:\n prompt_guard:\n enabled: true\n # model_id is auto-discovered from models directory:\n # - Legacy: models/jailbreak_classifier_modernbert-base_model\n # - LoRA: models/lora_jailbreak_classifier_bert_model (preferred)\n # models/lora_jailbreak_classifier_roberta_model\n # models/lora_jailbreak_classifier_modernbert_model\n threshold: 0.5\n use_cpu: false\n use_modernbert: true\n # jailbreak_mapping_path is auto-discovered from model directory\n"})}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Note:"})," The jailbreak classifier uses auto-discovery to find models in the ",(0,s.jsx)(n.code,{children:"models/"})," directory. The system prefers LoRA models (BERT > RoBERTa > ModernBERT) over legacy ModernBERT models for better accuracy."]}),"\n",(0,s.jsx)(n.h3,{id:"63-data-residency-and-compliance",children:"6.3 Data Residency and Compliance"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Considerations:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Semantic cache may store user queries"}),"\n",(0,s.jsx)(n.li,{children:"KV cache contains model activations"}),"\n",(0,s.jsx)(n.li,{children:"Distributed tracing may log request content"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Best Practices:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Cache Encryption:"})," Encrypt Milvus cache at rest and in transit"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"TTL Policies:"})," Automatic expiration of cached data (default: 2 hours)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Data Locality:"})," Deploy in compliance-approved regions"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Audit Logging:"})," Comprehensive logs for compliance audits"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Right to Deletion:"})," API for purging user data from caches"]}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h2,{id:"7-operational-considerations",children:"7. Operational Considerations"}),"\n",(0,s.jsx)(n.h3,{id:"71-monitoring-and-alerting",children:"7.1 Monitoring and Alerting"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Key Metrics:"})}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Metric"}),(0,s.jsx)(n.th,{children:"Threshold"}),(0,s.jsx)(n.th,{children:"Alert Severity"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"Semantic Router Latency (P99)"}),(0,s.jsx)(n.td,{children:"High"}),(0,s.jsx)(n.td,{children:"Warning"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"Dynamo Router Latency (P99)"}),(0,s.jsx)(n.td,{children:"High"}),(0,s.jsx)(n.td,{children:"Warning"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"Combined Latency (P99)"}),(0,s.jsx)(n.td,{children:"Very High"}),(0,s.jsx)(n.td,{children:"Critical"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"Semantic Cache Hit Rate"}),(0,s.jsx)(n.td,{children:"Low"}),(0,s.jsx)(n.td,{children:"Warning"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"KV Cache Hit Rate"}),(0,s.jsx)(n.td,{children:"Low"}),(0,s.jsx)(n.td,{children:"Warning"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"Security Block Rate"}),(0,s.jsx)(n.td,{children:"High"}),(0,s.jsx)(n.td,{children:"Warning"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"Error Rate"}),(0,s.jsx)(n.td,{children:"High"}),(0,s.jsx)(n.td,{children:"Critical"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"GPU Utilization"}),(0,s.jsx)(n.td,{children:"Too Low or Too High"}),(0,s.jsx)(n.td,{children:"Warning"})]})]})]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Dashboards:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Request Flow Dashboard:"})," Visualize request journey through layers"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Cache Performance Dashboard:"})," Hit rates, latency, eviction rates"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Security Dashboard:"})," PII detections, jailbreak blocks, audit logs"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Cost Dashboard:"})," Token usage, model selection, cost per query"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"73-failure-modes-and-recovery",children:"7.3 Failure Modes and Recovery"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Failure Scenario 1: Semantic Router Unavailable"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Detection:"})," Health check failures, timeout errors"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Impact:"})," No semantic routing, security filtering, or caching"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Recovery:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Envoy Gateway bypasses ExtProc (fallback mode)"}),"\n",(0,s.jsx)(n.li,{children:"Requests forwarded directly to Dynamo"}),"\n",(0,s.jsx)(n.li,{children:"Dynamo perform
1s default routing"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Mitigation:"})," Deploy 3+ replicas with anti-affinity"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Failure Scenario 2: Milvus Cache Unavailable"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Detection:"})," Connection errors, timeout"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Impact:"})," No semantic caching (cache misses)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Recovery:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Semantic Router continues with in-memory cache"}),"\n",(0,s.jsx)(n.li,{children:"All requests forwarded to Dynamo"}),"\n",(0,s.jsx)(n.li,{children:"Performance degradation but no outage"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Mitigation:"})," Milvus cluster deployment for HA"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Failure Scenario 3: Dynamo Frontend Unavailable"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Detection:"})," HTTP 503 errors, connection refused"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Impact:"})," No inference possible"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Recovery:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Envoy Gateway returns 503 to clients"}),"\n",(0,s.jsx)(n.li,{children:"Kubernetes restarts failed pods"}),"\n",(0,s.jsx)(n.li,{children:"Load balancer routes to healthy replicas"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Mitigation:"})," Deploy 2+ replicas with readiness probes"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Failure Scenario 4: Worker Pool Exhaustion"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Detection:"})," Queue depth alerts, high latency"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Impact:"})," Increased TTFT and ITL"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Recovery:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Dynamo Planner auto-scales workers"}),"\n",(0,s.jsx)(n.li,{children:"Semantic Router may route to alternative models"}),"\n",(0,s.jsx)(n.li,{children:"Requests queued until capacity available"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Mitigation:"})," Autoscaling policies, overprovisioning"]}),"\n"]}),"\n",(0,s.jsx)(n.h2,{id:"8-future-enhancements",children:"8. Future Enhancements"}),"\n",(0,s.jsx)(n.h3,{id:"81-advanced-routing-strategies",children:"8.1 Advanced Routing Strategies"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Multi-Objective Optimization:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Combine semantic quality, latency, and cost in routing decision"}),"\n",(0,s.jsx)(n.li,{children:"Pareto-optimal model selection"}),"\n",(0,s.jsx)(n.li,{children:"User-specified SLO preferences (fast vs. accurate vs. cheap)"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Adaptive Routing:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Learn from user feedback (thumbs up/down)"}),"\n",(0,s.jsx)(n.li,{children:"A/B testing of model selections"}),"\n",(0,s.jsx)(n.li,{children:"Reinforcement learning for routing policy"}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"82-cross-layer-optimization",children:"8.2 Cross-Layer Optimization"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Semantic-Aware KV Cache Management:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Prioritize KV cache retention for high-value categories"}),"\n",(0,s.jsx)(n.li,{children:"Semantic similarity for KV cache eviction decisions"}),"\n",(0,s.jsx)(n.li,{children:"Cross-request KV cache sharing for similar queries"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Predictive Prefetching:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Predict next query in conversation"}),"\n",(0,s.jsx)(n.li,{children:"Pre-warm KV cache for likely follow-ups"}),"\n",(0,s.jsx)(n.li,{children:"Speculative execution for low-latency responses"}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"83-multi-tenant-support",children:"8.3 Multi-Tenant Support"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Tenant Isolation:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Per-tenant semantic cache namespaces"}),"\n",(0,s.jsx)(n.li,{children:"Per-tenant model access policies"}),"\n",(0,s.jsx)(n.li,{children:"Per-tenant cost tracking and quotas"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Tenant-Specific Routing:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Custom model pools per tenant"}),"\n",(0,s.jsx)(n.li,{children:"Tenant-specific security policies"}),"\n",(0,s.jsx)(n.li,{children:"Tenant-specific SLOs"}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h2,{id:"9-references",children:"9. References"}),"\n",(0,s.jsx)(n.h3,{id:"91-nvidia-dynamo-documentation",children:"9.1 NVIDIA Dynamo Documentation"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://docs.nvidia.com/dynamo/latest/_sections/architecture.html",children:"Dynamo Architecture Overview"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://docs.nvidia.com/dynamo/latest/components/router/README.html",children:"Dynamo KV Router"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://docs.nvidia.com/dynamo/latest/_sections/disaggregated-serving.html",children:"Dynamo Disaggregated Serving"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://docs.nvidia.com/dynamo/latest/components/kvbm/README.html",children:"Dynamo KVBM"})}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"92-vllm-semantic-router-documentation",children:"9.2 vLLM Semantic Router Documentation"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://vllm-sr.ai/docs/overview/semantic-router-overview/",children:"Semantic Router Overview"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://vllm-sr.ai/docs/overview/architecture/system-architecture/",children:"System Architecture"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://vllm-sr.ai/docs/installation/k8s/ai-gateway",children:"Kubernetes Deployment"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://github.com/vllm-project/semantic-router/pull/322",children:"Distributed Tracing Support (PR #322)"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://vllm-sr.ai/docs/features/semantic-caching/",children:"Milvus-based Semantic Caching"})}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"93-related-research",children:"9.3 Related Research"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"DistServe:"})," Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Mooncake:"})," KVCache-centric Disaggregated Architecture for LLM Serving"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"RouteLLM:"})," Learning to Route LLMs with Preference Data"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"DeepSeek-V3:"})," Technical Report on Mixture-of-Experts Architecture"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"94-integration-proposals",children:"9.4 Integration Proposals"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://github.com/vllm-project/semantic-router/issues/295",children:"vLLM Production Stack Integration (#295)"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://vllm-sr.ai/docs/proposals/prompt-classification-routing/",children:"Prompt Classification Routing Proposal"})}),"\n"]}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h2,{id:"10-appendix",children:"10. Appendix"}),"\n",(0,s.jsx)(n.h3,{id:"101-glossary",children:"10.1 Glossary"}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Term"}),(0,s.jsx)(n.th,{children:"Definition"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"BERT"})}),(0,s.jsx)(n.td,{children:"Bidirectional Encoder Representations from Transformers"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"ExtProc"})}),(0,s.jsx)(n.td,{children:"Envoy External Processor (gRPC service for request processing)"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Fusion Routing"})}),(0,s.jsx)(n.td,{children:"Multi-signal routing combining BERT classification, keyword matching, and similarity search"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"ITL"})}),(0,s.jsx)(n.td,{children:"Inter-Token Latency (time between generated tokens)"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"KV Cache"})}),(0,s.jsx)(n.td,{children:"Key-Value cache storing transformer attention states"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"KVBM"})}),(0,s.jsx)(n.td,{children:"KV Block Manager (Dynamo component for cache management)"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Milvus"})}),(0,s.jsx)(n.td,{children:"Open-source vector database for semantic caching and similarity search"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"MoE"})}),(0,s.jsx)(n.td,{children:"Mixture-of-Experts (model architecture with specialized expert networks)"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"MoM"})}),(0,s.jsx)(n.td,{children:"Mixture-of-Models (routing to different models based on task)"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"NIXL"})}),(0,s.jsx)(n.td,{children:"NVIDIA Inference Transfer Library"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"OTLP"})}),(0,s.jsx)(n.td,{children:"OpenTelemetry Protocol (for distributed tracing and metrics)"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"PII"})}),(0,s.jsx)(n.td,{children:"Personally Identifiable Information"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Prompt Guard"})}),(0,s.jsx)(n.td,{children:"Jailbreak detection system using classification models to identify adversarial prompts"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"TTFT"})}),(0,s.jsx)(n.td,{children:"Time To First Token (latency until first token generated)"})]})]})]}),"\n",(0,s.jsx)(n.h3,{id:"102-system-prompt-examples",children:"10.2 System Prompt Examples"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Domain-Aware System Prompts for Key Categories:"})}),"\n",(0,s.jsxs)(n.p,{children:["The integration leverages ",(0,s.jsx)(n.strong,{children:"14 specialized system prompts"})," that are automatically injected based on query classification. Here are representative examples:"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"1. Math Category (Reasoning-Heavy)"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"You are a mathematics expert. Provide step-by-step solutions, show your\nwork clearly, and explain mathematical concepts in an understandable way.\n"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Purpose"}),": Encourage structured reasoning and clear explanations"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model"}),": DeepSeek-V3 (score: 1.0, reasoning: enabled)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"MoE Impact"}),": Activates mathematical reasoning experts"]}
1),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"2. Computer Science Category (Code-Focused)"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"You are a computer science expert with knowledge of algorithms, data structures,\nprogramming languages, and software engineering. Provide clear, practical solutions\nwith code examples when helpful.\n"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Purpose"}),": Balance theory with practical code examples"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model"}),": Qwen3 (score: 0.89, reasoning: disabled)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"MoE Impact"}),": Activates programming and algorithm experts"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"3. Business Category (Action-Oriented)"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"You are a senior business consultant and strategic advisor with expertise in\ncorporate strategy, operations management, financial analysis, marketing, and\norganizational development. Provide practical, actionable business advice backed\nby proven methodologies and industry best practices. Consider market dynamics,\ncompetitive landscape, and stakeholder interests in your recommendations.\n"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Purpose"}),": Emphasize actionable advice and business context"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model"}),": Phi-4 (score: 0.88, reasoning: disabled)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"MoE Impact"}),": Activates business strategy and analysis experts"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"4. Law Category (Disclaimer-Aware)"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"You are a knowledgeable legal expert with comprehensive understanding of legal\nprinciples, case law, statutory interpretation, and legal procedures. Provide\naccurate legal information while clearly stating that your responses are for\ninformational purposes only and do not constitute legal advice.\n"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Purpose"}),": Ensure accuracy while maintaining ethical boundaries"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model"}),": Phi-4 (score: 0.75, reasoning: disabled)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"MoE Impact"}),": Activates legal reasoning experts with appropriate disclaimers"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"5. Health Category (Evidence-Based)"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"You are a health and medical information expert with knowledge of anatomy,\nphysiology, diseases, treatments, preventive care, nutrition, and wellness.\nProvide accurate, evidence-based health information while emphasizing that\nyour responses are for educational purposes only and do not replace professional\nmedical advice.\n"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Purpose"}),": Balance informativeness with medical ethics"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Model"}),": Phi-4 (score: 0.76, reasoning: disabled)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"MoE Impact"}),": Activates medical knowledge experts with safety guardrails"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Complete Category List:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"math, computer science, physics, chemistry, biology, engineering"}),"\n",(0,s.jsx)(n.li,{children:"economics, business, law, psychology, philosophy, history, health, other"}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"System Prompt Benefits:"})}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"CoT Optimization"}),": Domain-specific reasoning patterns improve output quality"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Token Efficiency"}),": Focused prompts reduce unnecessary verbosity (10-15% token reduction)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"MoE Expert Matching"}),": Specialized terminology activates relevant experts (20-30% improvement in expert selection accuracy)"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Quality Control"}),": Category-specific disclaimers ensure ethical compliance"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"103-api-examples",children:"10.3 API Examples"}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Request with Semantic Router Headers:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'curl -X POST http://llm-gateway:8080/v1/chat/completions \\\n -H "Content-Type: application/json" \\\n -d \'{\n "model": "auto",\n "messages": [\n {\n "role": "user",\n "content": "Prove that the square root of 2 is irrational"\n }\n ]\n }\'\n'})}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Response with Routing Headers:"})}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-http",children:'HTTP/1.1 200 OK\nContent-Type: application/json\nx-vsr-selected-model: deepseek-v31\nx-vsr-selected-category: math\nx-vsr-selected-reasoning: on\nx-vsr-injected-system-prompt: true\nx-request-id: 7f3e9a2b4c5d6e8f\n\n{\n "id": "chatcmpl-abc123",\n "object": "chat.completion",\n "created": 1704067200,\n "model": "deepseek-v31",\n "choices": [\n {\n "index": 0,\n "message": {\n "role": "assistant",\n "content": "To prove that \u221a2 is irrational, we\'ll use proof by contradiction..."\n },\n "finish_reason": "stop"\n }\n ],\n "usage": {\n "prompt_tokens": 15,\n "completion_tokens": 250,\n "total_tokens": 265\n }\n}\n'})}),"\n",(0,s.jsx)(n.hr,{}),"\n",(0,s.jsx)(n.h2,{id:"conclusion",children:"Conclusion"}),"\n",(0,s.jsx)(n.p,{children:"This proposal outlines a comprehensive integration strategy between vLLM Semantic Router and NVIDIA Dynamo that combines semantic intelligence with infrastru
1cture optimization. The layered architecture ensures:"}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Semantic Correctness:"})," Right model selection based on query understanding"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Infrastructure Efficiency:"})," Optimal worker selection and KV cache utilization"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Security:"})," PII detection and jailbreak prevention before inference"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Performance:"})," Dual-layer caching for 40-60% latency reduction"]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Cost Optimization:"})," 55% cost reduction through intelligent routing"]}),"\n"]}),"\n",(0,s.jsxs)(n.p,{children:["The integration is designed to be ",(0,s.jsx)(n.strong,{children:"non-invasive"}),", ",(0,s.jsx)(n.strong,{children:"modular"}),", and ",(0,s.jsx)(n.strong,{children:"production-ready"}),", with clear implementation phases, comprehensive monitoring, and robust failure handling."]}),"\n",(0,s.jsx)(n.p,{children:(0,s.jsx)(n.strong,{children:"Next Steps:"})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsx)(n.li,{children:"Review and approve proposal"}),"\n",(0,s.jsx)(n.li,{children:"Begin Phase 1 implementation (Foundation)"}),"\n",(0,s.jsx)(n.li,{children:"Establish benchmark environment"}),"\n",(0,s.jsx)(n.li,{children:"Iterate based on performance results"}),"\n"]})]})}function x(e={}){const{wrapper:n}={...(0,t.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(h,{...e})}):h(e)}},12352(e,n,i){i.d(n,{A:()=>S});var r=i(96540),s=i(67489),t=i(12181),l=i(22007),c=i(10467),o=i(6342),a=i(95293);function d(){return h.apply(this,arguments)}function h(){return(h=(0,c.A)((0,l.A)().m(function e(){var n,r;return(0,l.A)().w(function(e){for(;;)switch(e.n){case 0:return e.n=1,Promise.all([i.e(70451),i.e(23923)]).then(()=>i(23923));case 1:return n=e.v.default,e.n=2,i.e(92227).then(()=>i(92227));case 2:r=e.v.default,n.registerLayoutLoaders(r);case 3:return e.a(2,n)}},e)}))).apply(this,arguments)}var x=null;function j(){return u.apply(this,arguments)}function u(){return(u=(0,c.A)((0,l.A)().m(function e(){return(0,l.A)().w(function(e){for(;;)if(0===e.n)return x||(x=d()),e.a(2,x)},e)}))).apply(this,arguments)}function m(){var e=(0,a.G)().colorMode,n=(0,o.p)().mermaid,i=n.theme[e],s=n.options;return(0,r.useMemo)(function(){return Object.assign({startOnLoad:!1},s,{theme:i})},[i,s])}function p(){return(p=(0,c.A)((0,l.A)().m(function e(n){var i,r,s,t,c,o;return(0,l.A)().w(function(e){for(;;)switch(e.p=e.n){case 0:return i=n.id,r=n.text,s=n.config,e.n=1,j();case 1:return(t=e.v).initialize(s),e.p=2,e.n=3,t.render(i,r);case 3:return e.a(2,e.v);case 4:throw e.p=4,o=e.v,null==(c=document.querySelector("#d"+i))||c.remove(),o;case 5:return e.a(2)}},e,null,[[2,4]])}))).apply(this,arguments)}function g(e){var n=e.text,i=e.config,s=(0,r.useState)(null),t=s[0],l=s[1],c=(0,r.useState)("mermaid-svg-"+Math.round(1e7*Math.random()))[0],o=m(),a=null!=i?i:o;return(0,r.useEffect)(function(){(function(e){return p.apply(this,arguments)})({id:c,text:n,config:a}).then(l).catch(function(e){l(function(){throw e})})},[c,n,a]),t}const y="container_lyt7";var f=i(74848);function v(e){var n=e.renderResult,i=(0,r.useRef)(null);return(0,r.useEffect)(function(){var e=i.current;null==n.bindFunctions||n.bindFunctions(e)},[n]),(0,f.jsx)("div",{ref:i,className:"docusaurus-mermaid-container "+y,dangerouslySetInnerHTML:{__html:n.svg}})}function b(e){var n=g({text:e.value});return null===n?null:(0,f.jsx)(v,{renderResult:n})}function S(e){return(0,f.jsx)(s.A,{fallback:function(e){return(0,f.jsx)(t.MN,Object.assign({},e))},children:(0,f.jsx)(b,Object.assign({},e))})}},2107(e,n,i){i.d(n,{A:()=>o});var r=i(96540),s=i(40961),t=i(12352);const l={mermaidContainer:"mermaidContainer_iBhF",zoomHint:"zoomHint_vHSF",modal:"modal_QBuq",fadeIn:"fadeIn_fBYh",modalContent:"modalContent_vaBQ",slideIn:"slideIn_tWsR",modalHeader:"modalHeader_CROH",modalTitle:"modalTitle_xX6M",modalControls:"modalControls_xcR0",zoomIndicator:"zoomIndicator_erY9",zoomButton:"zoomButton_CYM3",resetButton:"resetButton_tuBT",closeButton:"closeButton_jL6V",modalBody:"modalBody_a9z5",diagramContainer:"diagramContainer_lCnz",mermaid:"mermaid_tdwO"};var c=i(74848);const o=function(e){var n=e.children,i=e.title,o=e.defaultZoom,a=void 0===o?1.2:o,d=(0,r.useState)(!1),h=d[0],x=d[1],j=(0,r.useState)(!1),u=j[0],m=j[1],p=(0,r.useState)(a),g=p[0],y=p[1],f=(0,r.useRef)(null),v=(0,r.useRef)(null),b=(0,r.useCallback)(function(){x(!0),y(a),document.body.style.overflow="hidden"},[a]),S=(0,r.useCallback)(function(){x(!1),document.body.style.overflow="unset",v.current&&v.current.focus()},[]),k=(0,r.useCallback)(function(){y(function(e){return Math.min(e+.2,5)})},[]),R=(0,r.useCallback)(function(){y(function(e){return Math.max(e-.2,.5)})},[]),w=(0,r.useCallback)(function(){y(a)},[a]);(0,r.useEffect)(function(){var e=function(e){"Escape"===e.key&&h&&S()},n=function(e){f.current&&!f.current.contains(e.target)&&S()},i=function(e){h&&("="===e.key||"+"===e.key?(e.preventDefault(),k()):"-"===e.key?(e.preventDefault(),R()):"0"===e.key&&(e.preventDefault(),w()))};
1return h&&(document.addEventListener("keydown",e),document.addEventListener("mousedown",n),document.addEventListener("keydown",i),setTimeout(function(){f.current&&f.current.focus()},100)),function(){document.removeEventListener("keydown",e),document.removeEventListener("mousedown",n),document.removeEventListener("keydown",i)}},[h,S,k,R,w]),(0,r.useEffect)(function(){return function(){document.body.style.overflow="unset"}},[]);var P=(0,c.jsx)("div",{className:l.modal,role:"dialog","aria-modal":"true","aria-labelledby":i?"modal-title":void 0,"aria-describedby":"modal-description",children:(0,c.jsxs)("div",{className:l.modalContent,ref:f,tabIndex:-1,children:[(0,c.jsxs)("div",{className:l.modalHeader,children:[i&&(0,c.jsx)("h3",{id:"modal-title",className:l.modalTitle,children:i}),(0,c.jsxs)("div",{className:l.modalControls,children:[(0,c.jsxs)("span",{className:l.zoomIndicator,children:[Math.round(100*g),"%"]}),(0,c.jsx)("button",{className:l.zoomButton,onClick:R,disabled:g<=.5,"aria-label":"Reduce the size of the chart",type:"button",title:"Reduce (Shortcut key: -)",children:(0,c.jsxs)("svg",{width:"18",height:"18",viewBox:"0 0 24 24",fill:"none",stroke:"currentColor",strokeWidth:"2",children:[(0,c.jsx)("circle",{cx:"11",cy:"11",r:"8"}),(0,c.jsx)("path",{d:"M8 11h6"}),(0,c.jsx)("path",{d:"m21 21-4.35-4.35"})]})}),(0,c.jsx)("button",{className:l.resetButton,onClick:w,"aria-label":"Reset to default zoom level "+Math.round(100*a)+"%",type:"button",title:"Reset to default zoom level "+Math.round(100*a)+"% (Shortcut key: 0)",children:(0,c.jsxs)("svg",{width:"18",height:"18",viewBox:"0 0 24 24",fill:"none",stroke:"currentColor",strokeWidth:"2",children:[(0,c.jsx)("path",{d:"M3 3l18 18"}),(0,c.jsx)("path",{d:"m19 4-7 7-7-7"}),(0,c.jsx)("path",{d:"m5 20 7-7 7 7"})]})}),(0,c.jsx)("button",{className:l.zoomButton,onClick:k,disabled:g>=5,"aria-label":"Enlarge the chart",type:"button",title:"Enlarge (Shortcut key: +)",children:(0,c.jsxs)("svg",{width:"18",height:"18",viewBox:"0 0 24 24",fill:"none",stroke:"currentColor",strokeWidth:"2",children:[(0,c.jsx)("circle",{cx:"11",cy:"11",r:"8"}),(0,c.jsx)("path",{d:"M8 11h6"}),(0,c.jsx)("path",{d:"M11 8v6"}),(0,c.jsx)("path",{d:"m21 21-4.35-4.35"})]})}),(0,c.jsx)("button",{className:l.closeButton,onClick:S,"aria-label":"Close the zoomed view",type:"button",children:(0,c.jsxs)("svg",{width:"18",height:"18",viewBox:"0 0 24 24",fill:"none",stroke:"currentColor",strokeWidth:"2",children:[(0,c.jsx)("line",{x1:"18",y1:"6",x2:"6",y2:"18"}),(0,c.jsx)("line",{x1:"6",y1:"6",x2:"18",y2:"18"})]})})]})]}),(0,c.jsx)("div",{className:l.modalBody,id:"modal-description","aria-label":"Enlarged Mermaid diagram",children:(0,c.jsx)("div",{className:l.diagramContainer,style:{transform:"scale("+g+")",transformOrigin:"center top"},children:(0,c.jsx)(t.A,{value:n})})})]})});return(0,c.jsxs)(c.Fragment,{children:[(0,c.jsxs)("div",{ref:v,className:l.mermaidContainer+" "+(u?l.hovered:""),onClick:b,onMouseEnter:function(){return m(!0)},onMouseLeave:function(){return m(!1)},role:"button",tabIndex:0,onKeyDown:function(e){"Enter"!==e.key&&" "!==e.key||(e.preventDefault(),b())},"aria-label":"Click to enlarge "+(i||"Mermaid diagram"),"aria-expanded":h,children:[(0,c.jsxs)("div",{className:l.zoomHint,"aria-hidden":"true",children:[(0,c.jsxs)("svg",{width:"20",height:"20",viewBox:"0 0 24 24",fill:"none",stroke:"currentColor",strokeWidth:"2",children:[(0,c.jsx)("circle",{cx:"11",cy:"11",r:"8"}),(0,c.jsx)("path",{d:"m21 21-4.35-4.35"}),(0,c.jsx)("path",{d:"M11 8v6"}),(0,c.jsx)("path",{d:"M8 11h6"})]}),(0,c.jsx)("span",{children:"Click to enlarge"})]}),(0,c.jsx)(t.A,{value:n})]}),h&&(0,s.createPortal)(P,document.body)]})}},28453(e,n,i){i.d(n,{R:()=>l,x:()=>c});var r=i(96540);const s={},t=r.createContext(s);function l(e){const n=r.useContext(t);return r.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function c(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:l(e.components),r.createElement(t.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.