https://llm-d.ai/assets/js/ba57124e.527b5bff.js

1"use strict";(globalThis.webpackChunkllm_d_website=globalThis.webpackChunkllm_d_website||[]).push([[36092],{65703(e,n,r){r.r(n),r.d(n,{assets:()=>c,contentTitle:()=>d,default:()=>a,frontMatter:()=>l,metadata:()=>t,toc:()=>o});const t=JSON.parse('{"id":"operations/observability/promql","title":"PromQL Query Reference","description":"Ready-to-use PromQL queries for monitoring llm-d deployments. Use these in the Prometheus UI or as the basis for Grafana panels.","source":"@site/versioned_docs/version-0.8/operations/observability/promql.md","sourceDirName":"operations/observability","slug":"/operations/observability/promql","permalink":"/docs/0.8/operations/observability/promql","draft":false,"unlisted":false,"tags":[],"version":"0.8","frontMatter":{},"sidebar":"docsSidebar","previous":{"title":"Tracing","permalink":"/docs/0.8/operations/observability/tracing"},"next":{"title":"Observability Setup","permalink":"/docs/0.8/operations/observability/setup"}}');var s=r(74848),i=r(28453);const l={},d="PromQL Query Reference",c={},o=[{value:"Tier 1: Immediate Failure & Saturation Indicators",id:"tier-1-immediate-failure--saturation-indicators",level:2},{value:"Tier 2: Diagnostic Drill-Down",id:"tier-2-diagnostic-drill-down",level:2},{value:"Basic Model Serving",id:"basic-model-serving",level:3},{value:"Routing & Load Balancing",id:"routing--load-balancing",level:3},{value:"Prefix Caching",id:"prefix-caching",level:3},{value:"Prefill/Decode Disaggregation",id:"prefilldecode-disaggregation",level:3},{value:"Flow Control",id:"flow-control",level:3},{value:"Notes",id:"notes",level:2}];function h(e){const n={a:"a",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",...(0,i.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.header,{children:(0,s.jsx)(n.h1,{id:"promql-query-reference",children:"PromQL Query Reference"})}),"\n",(0,s.jsx)(n.p,{children:"Ready-to-use PromQL queries for monitoring llm-d deployments. Use these in the Prometheus UI or as the basis for Grafana panels."}),"\n",(0,s.jsxs)(n.p,{children:["To generate traffic and populate error metrics for testing, use the ",(0,s.jsx)(n.a,{href:"https://github.com/llm-d/llm-d/tree/main/guides/recipes/observability/generate-traffic-basic.sh",children:"traffic generation script"}),"."]}),"\n",(0,s.jsx)(n.h2,{id:"tier-1-immediate-failure--saturation-indicators",children:"Tier 1: Immediate Failure & Saturation Indicators"}),"\n",(0,s.jsx)(n.p,{children:"Start here when something looks wrong."}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Metric Need"}),(0,s.jsx)(n.th,{children:"PromQL Query"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Overall Error Rate"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum(rate(llm_d_epp_request_error_total[5m])) / sum(rate(llm_d_epp_request_total[5m]))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Per-Model Error Rate"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(model_name) (rate(llm_d_epp_request_error_total[5m])) / sum by(model_name) (rate(llm_d_epp_request_total[5m]))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Request Preemptions"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(pod, instance) (rate(vllm:num_preemptions[5m]))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Overall Latency P90"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"histogram_quantile(0.90, sum by(le) (rate(llm_d_epp_request_duration_seconds_bucket[5m])))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Overall Latency P99"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"histogram_quantile(0.99, sum by(le) (rate(llm_d_epp_request_duration_seconds_bucket[5m])))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"TTFT P99 per model"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"histogram_quantile(0.99, sum by(le, model_name) (rate(vllm:time_to_first_token_seconds_bucket[5m])))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Inter-Token Latency P99"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"histogram_quantile(0.99, sum by(le, model_name) (rate(vllm:inter_token_latency_seconds_bucket[5m])))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Request Rate"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(model_name) (rate(llm_d_epp_request_total[5m]))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"GPU Utilization"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"avg by(gpu, node) (DCGM_FI_DEV_GPU_UTIL or nvidia_gpu_duty_cycle)"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"EPP E2E Latency P99"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"histogram_quantile(0.99, sum by(le) (rate(llm_d_epp_scheduler_e2e_duration_seconds_bucket[5m])))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"EPP Plugin Latency P99"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"histogram_quantile(0.99, sum by(le, plugin_type) (rate(llm_d_epp_plugin_duration_seconds_bucket[5m])))"})})]})]})]}),"\n",(0,s.jsx)(n.h2,{id:"tier-2-diagnostic-drill-down",children:"Tier 2: Diagnostic Drill-Down"}),"\n",(0,s.jsx)(n.h3,{id:"basic-model-serving",children:"Basic Model Serving"}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Metric Need"}),(0,s.jsx)(n.th,{children:"PromQL Query"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"KV Cache Utilization"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"avg by(pod, model_name) (vllm:kv_cache_usage_perc)"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Request Queue Depth"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(pod, model_name) (vllm:num_requests_waiting)"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Active Requests"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"avg by(pod) (vllm:num_requests_running)"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsxs)(n.td,{children:[(0,s.jsx)(n.strong,{children:"Total Throughput"})," (tokens/sec)"]}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(model_name, pod) (rate(vllm:prompt_tokens_total[5m]) + rate(vllm:generation_tokens_total[5m]))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Generation Token Rate"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(model_name, pod) (rate(vllm:generation_tokens_total[5m]))"})})]})]})]}),"\n",(0,s.jsx)(n.h3,{id:"routing--load-balancing",children:"Routing & Load Balancing"}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Metric Need"}),(0,s.jsx)(n.th,{children:"PromQL Query"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"QPS per pod"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(pod) (rate(llm_d_epp_request_total[5m]))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Token distribution per pod"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(pod) (rate(vllm:prompt_tokens_total[5m]) + rate(vllm:generation_tokens_total[5m]))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Routing decision latency P99"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"histogram_quantile(0.99, sum by(le) (rate(llm_d_epp_plugin_duration_seconds_bucket[5m])))"})})]})]})]}),"\n",(0,s.jsx)(n.h3,{id:"prefix-caching",children:"Prefix Caching"}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Metric Need"}),(0,s.jsx)(n.th,{children:"PromQL Query"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Cache hit rate"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum(rate(vllm:prefix_cache_hits_total[5m])) / sum(rate(vllm:prefix_cache_queries_total[5m]))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Per-pod hit rate"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(pod) (rate(vllm:prefix_cache_hits_total[5m])) / sum by(pod) (rate(vllm:prefix_cache_queries_total[5m]))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"EPP prefix indexer size"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"llm_d_epp_prefix_indexer_size"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"EPP prefix hit ratio P90"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"histogram_quantile(0.90, sum by(le) (rate(llm_d_epp_prefix_indexer_hit_ratio_bucket[5m])))"})})]})]})]}),"\n",(0,s.jsx)(n.h3,{id:"prefilldecode-disaggregation",children:"Prefill/Decode Disaggregation"}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Metric Need"}),(0,s.jsx)(n.th,{children:"PromQL Query"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Prefill worker utilization"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:'avg by(pod) (vllm:num_requests_running{pod=~".*prefill.*"})'})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Decode KV cache utilization"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:'avg by(pod) (vllm:kv_cache_usage_perc{pod=~".*decode.*"})'})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"P/D decision ratio"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:'sum(rate(llm_d_epp_pd_decision_total{decision_type="prefill-decode"}[5m])) / sum(rate(llm_d_epp_pd_decision_total[5m]))'})})]})]})]}),"\n",(0,s.jsx)(n.h3,{id:"flow-control",children:"Flow Control"}),"\n",(0,s.jsxs)(n.p,{children:["Requires the ",(0,s.jsx)(n.code,{children:"flowControl"})," feature gate enabled on the EPP."]}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Metric Need"}),(0,s.jsx)(n.th,{children:"PromQL Query"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Queue size"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum(llm_d_epp_flow_control_queue_size)"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Queue size by priority"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"sum by(priority) (llm_d_epp_flow_control_queue_size)"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Queue wait time P99"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"histogram_quantile(0.99, sum by(le) (rate(llm_d_epp_flow_control_request_queue_duration_seconds_bucket[5m])))"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.strong,{children:"Pool saturation"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"llm_d_epp_flow_control_pool_saturation"})})]})]})]}),"\n",(0,s.jsx)(n.h2,{id:"notes",children:"Notes"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Metric name prefixes:"})," Current deployments use ",(0,s.jsx)(n.code,{children:"llm_d_epp_*"}),". Older deployments may use ",(0,s.jsx)(n.code,{children:"llm_d_router_epp_*"}),", ",(0,s.jsx)(n.code,{children:"inference_objective_*"})," or ",(0,s.jsx)(n.code,{children:"inference_extension_*"}),' \u2014 update accordingly if panels show "No data".']}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Histograms:"})," Always include ",(0,s.jsx)(n.code,{children:"by(le)"})," when using ",(0,s.jsx)(n.code,{children:"histogram_quantile()"}),":"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-promql",children:"histogram_quantile(0.99, sum by(le) (rate(metric_name_bucket[5m])))\n"})}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Error metrics"})," only appear after the first error occurs. Use the ",(0,s.jsx)(n.a,{href:"https://github.com/llm-d/llm-d/tree/main/guides/recipes/observability/generate-traffic-basic.sh",children:"traffic generation script"})," to populate them for testing."]})]})}function a(e={}){const{wrapper:n}={...(0,i.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(h,{...e})}):h(e)}},28453(e,n,r){r.d(n,{R:()=>l,x:()=>d});var t=r(96540);const s={},i=t.createContext(s);function l(e){const n=t.useContext(i);return t.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function d(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:l(e.components),t.createElement(i.Provider,{value:n},e.children)}}}]);