1"use strict";(globalThis.webpackChunkllm_d_website=globalThis.webpackChunkllm_d_website||[]).push([[10087],{46875(e,s,t){t.r(s),t.d(s,{assets:()=>o,contentTitle:()=>i,default:()=>h,frontMatter:()=>c,metadata:()=>r,toc:()=>d});const r=JSON.parse('{"id":"architecture/advanced/batch/async-processor","title":"Async Processor Architecture","description":"The Async Processor is a lightweight dispatch agent that pulls inference requests from message queues and forwards them to the llm-d Router. It uses dispatch gates to regulate dispatch rate based on system metrics, ensuring that the dispatched workloads don\'t overflow the inference servers.","source":"@site/versioned_docs/version-0.8/architecture/advanced/batch/async-processor.md","sourceDirName":"architecture/advanced/batch","slug":"/architecture/advanced/batch/async-processor","permalink":"/docs/0.8/architecture/advanced/batch/async-processor","draft":false,"unlisted":false,"tags":[],"version":"0.8","frontMatter":{},"sidebar":"docsSidebar","previous":{"title":"Batch Gateway Architecture","permalink":"/docs/0.8/architecture/advanced/batch/batch-gateway"},"next":{"title":"Operational Excellence","permalink":"/docs/0.8/operations/"}}');var n=t(74848),a=t(28453);const c={},i="Async Processor Architecture",o={},d=[{value:"How It Works",id:"how-it-works",level:2},{value:"Dispatch Gates",id:"dispatch-gates",level:2},{value:"Message Queue Integrations",id:"message-queue-integrations",level:2},{value:"Concurrency and Retries",id:"concurrency-and-retries",level:2},{value:"Observability",id:"observability",level:2},{value:"Related",id:"related",level:2}];function l(e){const s={a:"a",code:"code",h1:"h1",h2:"h2",header:"header",li:"li",ol:"ol",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(s.header,{children:(0,n.jsx)(s.h1,{id:"async-processor-architecture",children:"Async Processor Architecture"})}),"\n",(0,n.jsx)(s.p,{children:"The Async Processor is a lightweight dispatch agent that pulls inference requests from message queues and forwards them to the llm-d Router. It uses dispatch gates to regulate dispatch rate based on system metrics, ensuring that the dispatched workloads don't overflow the inference servers."}),"\n",(0,n.jsx)(s.h2,{id:"how-it-works",children:"How It Works"}),"\n",(0,n.jsxs)(s.ol,{children:["\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Poll"})," \u2014 workers pull requests from one or more message queues."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Gate"})," \u2014 before dispatching, each request passes through a dispatch gate that checks whether the system has capacity. If the gate is closed (budget = 0), the request waits."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Dispatch"})," \u2014 the worker sends an HTTP request to the llm-d Router with deadline propagation."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Result"})," \u2014 on success, results are written back to a queue. On retryable failure (rate limiting, transient errors), the request is re-queued with exponential backoff."]}),"\n"]}),"\n",(0,n.jsx)(s.h2,{id:"dispatch-gates",children:"Dispatch Gates"}),"\n",(0,n.jsx)(s.p,{children:"The dispatch gate controls the rate by which the processor sends requests. Each queue can have its own gate, allowing independent dispatch control per workload."}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,n.jsxs)(s.table,{children:[(0,n.jsx)(s.thead,{children:(0,n.jsxs)(s.tr,{children:[(0,n.jsx)(s.th,{children:"Gate type"}),(0,n.jsx)(s.th,{children:"Behavior"})]})}),(0,n.jsxs)(s.tbody,{children:[(0,n.jsxs)(s.tr,{children:[(0,n.jsx)(s.td,{children:(0,n.jsx)(s.code,{children:"constant"})}),(0,n.jsx)(s.td,{children:"Always open \u2014 no throttling."})]}),(0,n.jsxs)(s.tr,{children:[(0,n.jsx)(s.td,{children:(0,n.jsx)(s.code,{children:"redis"})}),(0,n.jsx)(s.td,{children:"Reads a budget value from a Redis key, allowing external systems to control dispatch rate."})]}),(0,n.jsxs)(s.tr,{children:[(0,n.jsx)(s.td,{children:(0,n.jsx)(s.code,{children:"prometheus-saturation"})}),(0,n.jsx)(s.td,{children:"Queries Prometheus for model server saturation metrics. Dispatches when saturation is below a configurable threshold."})]}),(0,n.jsxs)(s.tr,{children:[(0,n.jsx)(s.td,{children:(0,n.jsx)(s.code,{children:"prometheus-budget"})}),(0,n.jsx)(s.td,{children:"Computes available capacity from downstream metrics."})]})]})]}),"\n",(0,n.jsx)(s.h2,{id:"message-queue-integrations",children:"Message Queue Integrations"}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,n.jsxs)(s.table,{children:[(0,n.jsx)(s.thead,{children:(0,n.jsxs)(s.tr,{children:[(0,n.jsx)(s.th,{children:"Implementation"}),(0,n.jsx)(s.th,{children:"Characteristics"})]})}),(0,n.jsxs)(s.tbody,{children:[(0,n.jsxs)(s.tr,{children:[(0,n.jsx)(s.td,{children:"Redis Sorted Set"}),(0,n.jsx)(s.td,{children:"Persisted, priority-ordered by deadline. Supports per-queue gate configuration."})]}),(0,n.jsxs)(s.tr,{children:[(0,n.jsx)(s.td,{children:"Redis Pub/Sub"}),(0,n.jsx)(s.td,{children:"Ephemeral, fan-out delivery. Single global gate."})]}),(0,n.jsxs)(s.tr,{children:[(0,n.jsx)(s.td,{children:"GCP Pub/Sub"}),(0,n.jsx)(s.td,{children:"Cloud-native, scalable. Supports per-subscription gating."})]})]})]}),"\n",(0,n.jsx)(s.h2,{id:"concurrency-and-retries",children:"Concurrency and Retries"}),"\n",(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Worker pool"})," \u2014 configurable number of concurrent workers (default 8) process requests in parallel."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Deadline enforcement"})," \u2014 each request carries a deadline from the queue message. Workers abandon requests that cannot complete before their deadline."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Exponential backoff"}
1)," \u2014 retryable failures are re-queued with backoff (base 2s, max 60s, with jitter). Fatal errors (bad payload, unrecoverable failures) are not retried."]}),"\n"]}),"\n",(0,n.jsx)(s.h2,{id:"observability",children:"Observability"}),"\n",(0,n.jsx)(s.p,{children:"Prometheus metrics include request totals, success/failure counts, retry counts, deadline-exceeded counts, shedded request counts, and request latency histograms."}),"\n",(0,n.jsx)(s.h2,{id:"related",children:"Related"}),"\n",(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.a,{href:"/docs/0.8/well-lit-paths/workloads/batch-serving/asynchronous-processing",children:"Async Processor Well-Lit Path"})," \u2014 a guide for deploying the Async Processor."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.a,{href:"https://github.com/llm-d-incubation/llm-d-async",children:"Async Processor Repository"})," \u2014 source code and Helm chart."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.a,{href:"/docs/0.8/architecture/advanced/batch/batch-gateway",children:"Batch Gateway"})," \u2014 composes with the Async Processor for batch job management."]}),"\n"]})]})}function h(e={}){const{wrapper:s}={...(0,a.R)(),...e.components};return s?(0,n.jsx)(s,{...e,children:(0,n.jsx)(l,{...e})}):l(e)}},28453(e,s,t){t.d(s,{R:()=>c,x:()=>i});var r=t(96540);const n={},a=r.createContext(n);function c(e){const s=r.useContext(a);return r.useMemo(function(){return"function"==typeof e?e(s):{...s,...e}},[s,e])}function i(e){let s;return s=e.disableParentContext?"function"==typeof e.components?e.components(n):e.components||n:c(e.components),r.createElement(a.Provider,{value:s},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.