1"use strict";(self.webpackChunksemantic_router_docs=self.webpackChunksemantic_router_docs||[]).push([[18816],{92388(e,n,o){o.r(n),o.d(n,{assets:()=>a,contentTitle:()=>s,default:()=>h,frontMatter:()=>i,metadata:()=>r,toc:()=>d});const r=JSON.parse('{"id":"fleet-sim/power-model","title":"Power Model","description":"Fleet Sim can add an estimated power curve to a calibrated performance","source":"@site/docs/fleet-sim/power-model.md","sourceDirName":"fleet-sim","slug":"/fleet-sim/power-model","permalink":"/docs/fleet-sim/power-model","draft":false,"unlisted":false,"editUrl":"https://github.com/vllm-project/semantic-router/edit/main/website/docs/fleet-sim/power-model.md","tags":[],"version":"current","frontMatter":{"title":"Power Model"},"sidebar":"tutorialSidebar","previous":{"title":"Simulation Model","permalink":"/docs/fleet-sim/sim-algorithms"},"next":{"title":"Printable Guide","permalink":"/docs/fleet-sim/guide"}}');var t=o(74848),l=o(28453);const i={title:"Power Model"},s="Power model",a={},d=[{value:"Power-model variants",id:"power-model-variants",level:2},{value:"Manual profile",id:"manual-profile",level:3},{value:"Computed profile",id:"computed-profile",level:3},{value:"Built-in profile boundary",id:"built-in-profile-boundary",level:2},{value:"Tokens per watt",id:"tokens-per-watt",level:2},{value:"Run an energy study",id:"run-an-energy-study",level:2},{value:"Concurrency-cap analysis",id:"concurrency-cap-analysis",level:2},{value:"Calibrate a profile",id:"calibrate-a-profile",level:2},{value:"Reporting checklist",id:"reporting-checklist",level:2}];function c(e){const n={code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",ol:"ol",p:"p",pre:"pre",ul:"ul",...(0,l.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(n.header,{children:(0,t.jsx)(n.h1,{id:"power-model",children:"Power model"})}),"\n",(0,t.jsx)(n.p,{children:"Fleet Sim can add an estimated power curve to a calibrated performance\nprofile. This supports two planning questions:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"how output tokens per joule change with pool composition and utilization;"}),"\n",(0,t.jsx)(n.li,{children:"how a concurrency cap may trade modeled board power for queueing latency."}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:"Power results are estimates. The DES can check the latency side of a scenario,\nbut it does not measure or independently validate the power curve."}),"\n",(0,t.jsx)(n.h2,{id:"power-model-variants",children:"Power-model variants"}),"\n",(0,t.jsx)(n.h3,{id:"manual-profile",children:"Manual profile"}),"\n",(0,t.jsxs)(n.p,{children:["A ",(0,t.jsx)(n.code,{children:"ManualProfile"})," can define:"]}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"power_idle_w"}),": modeled board power at the low-concurrency endpoint;"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"power_nominal_w"}),": modeled board power at the high-concurrency endpoint;"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"power_logistic_k"}),": curve steepness; and"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"power_logistic_x0"}),": midpoint on a ",(0,t.jsx)(n.code,{children:"log2(concurrency)"})," axis."]}),"\n"]}),"\n",(0,t.jsxs)(n.p,{children:["When ",(0,t.jsx)(n.code,{children:"power_logistic_k"})," is greater than zero, power at concurrency ",(0,t.jsx)(n.code,{children:"b"})," is:"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-text",children:"P_range = P_nominal - P_idle\nP(b) = P_idle + P_range / (1 + exp(-k * (log2(max(1, b)) - x0)))\n"})}),"\n",(0,t.jsxs)(n.p,{children:["When ",(0,t.jsx)(n.code,{children:"k"})," is zero, the model linearly interpolates between ",(0,t.jsx)(n.code,{children:"P_idle"})," and\n",(0,t.jsx)(n.code,{children:"P_nominal"})," using ",(0,t.jsx)(n.code,{children:"b / max_slots"}),"."]}),"\n",(0,t.jsx)(n.p,{children:"Both variants use active sequences as the load variable. They do not model\nclock state, temperature, kernel mix, host power, networking, or cooling."}),"\n",(0,t.jsx)(n.h3,{id:"computed-profile",children:"Computed profile"}),"\n",(0,t.jsxs)(n.p,{children:["A ",(0,t.jsx)(n.code,{children:"ComputedProfile"})," estimates power from the hardware TDP and the profile's KV\ntraffic and tensor-core activity. It interpolates between fixed fractions of\nTDP as modeled activity rises."]}),"\n",(0,t.jsx)(n.p,{children:"This is a coarse transfer model, not a substitute for a batch-versus-power\nmeasurement on the target GPU. Applying the same TDP fractions to another\narchitecture, model, precision, or parallel layout adds uncertainty even when\nthe hardware specification itself is correct."}),"\n",(0,t.jsx)(n.h2,{id:"built-in-profile-boundary",children:"Built-in profile boundary"}),"\n",(0,t.jsxs)(n.p,{children:["The CLI includes manual profiles named ",(0,t.jsx)(n.code,{children:"h100"}),", ",(0,t.jsx)(n.code,{children:"a100"}),", and ",(0,t.jsx)(n.code,{children:"a10g"}),". They make\nthe examples runnable, but their constants should be treated as editable\nplanning assumptions:"]}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"their prices are static values embedded in source, not a live cloud-price\nfeed;"}),"\n",(0,t.jsx)(n.li,{children:"their performance and power curves are not measurements of your deployment;"}),"\n",(0,t.jsx)(n.li,{children:"the H100/A100 profiles are intended to represent a 70B-class, multi-GPU\nlayout, while A10G represents a smaller, single-GPU model; and"}),"\n",(0,t.jsx)(n.li,{children:"a profile name does not encode every serving parameter that affects power."}),"\n"]}),"\n",(0,t.jsxs)(n.p,{children:["Consequently, a default ",(0,t.jsx)(n.code,{children:"h100"})," versus ",(0,t.jsx)(n.code,{children:"a10g"})," tokens-per-watt result compares\ntwo modeled systems, including model size. It does not establish that one GPU\nis more efficient for the same model."]}),"\n",(0,t.jsx)(n.h2,{id:"tokens-per-watt",children:"Tokens per watt"}),"\n",(0,t.jsx)(n.p,{children:"For one homogeneous pool, Fleet Sim estimates output throughput from arrival\nrate and mean output length, then divides by modeled pool power:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-text",children:"pool_output_tokens_per_second = lambda_pool * mean_output_tokens\npool_power_watts = N_pool * P(mean_active_sequences)\npool_tokens_per_watt = pool_output_tokens_per_second / pool_power_watts\n"})}),"\n",(0,t.jsx)(n.p,{children:"For a multi-pool fleet, the correct aggregate is:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-text",children:"fleet_tokens_per_watt =\n sum(lambda_i * mean_output_tokens_i) /\n sum(N_i * P_i(mean_active_sequences_i))\n"})}),"\n",(0,t.jsx)(n.p,{children:"Fleet size can cancel algebraically for one homogeneous pool at a fixed\noperating point. It does not cancel across pools with different sizes, traffic\nfractions, or power curves."}),"\n",(0,t.jsx)(n.p,{children:"The CDF workflow estimates mean output length using its default 20% share of\ntotal tokens. If your output ratio differs, tokens-per-watt can move even when\nthe power curve does not."}),"\n",(0,t.jsx)(n.h2,{id:"run-an-energy-study",children:"Run an energy study"}),"\n",(0,t.jsx)(n.p,{children:"Single-pool mode applies the full workload to each selected profile:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-bash",children:"vllm-sr-sim tok-per-watt \\\n --cdf data/azure_cdf.json \\\n --lam 200 \\\n --slo 500 \\\n --gpus h100 a100 \\\n --rho-sweep \\\n --out energy.json\n"})}),"\n",(0,t.jsx)(n.p,{children:"Use this as a hardware comparison only after creating profiles for the same\nmodel, precision, parallel layout, context, and vLLM configuration."}),"\n",(0,t.jsx)(n.p,{children:"Two-pool mode compares a routed topology with a homogeneous long-pool\nbaseline:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-bash",children:"vllm-sr-sim tok-per-watt \\\n --cdf data/azure_cdf.json \\\n --lam 200 \\\n --slo 500 \\\n --b-short 6144 \\\n --gpu-short a10g \\\n --gpu-long h100 \\\n --out routed-energy.json\n"})}),"\n",(0,t.jsx)(n.p,{children:"In this form, a model switch can be intentional. Report it as a combined\nrouting, model, and hardware comparison, and validate answer quality\nseparately."}),"\n",(0,t.jsx)(n.h2,{id:"concurrency-cap-analysis",children:"Concurrency-cap analysis"}),"\n",(0,t.jsxs)(n.p,{children:[(0,t.jsx)(n.code,{children:"grid-flex"})," evaluates a fixed fleet while lowering its modeled concurrency\nlimit:"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-bash",children:"vllm-sr-sim grid-flex \\\n --cdf data/azure_cdf.json \\\n --lam 200 \\\n --n-gpus 32 \\\n --gpu h100 \\\n --slo 500 \\\n --flex-pcts 0 10 20 30 \\\n --verify-des 20000 \\\n --out flex.json\n"})}),"\n",(0,t.jsx)(n.p,{children:"For each requested reduction, Fleet Sim:"}),"\n",(0,t.jsxs)(n.ol,{children:["\n",(0,t.jsxs)(n.li,{children:["computes a target per-profile power relative to ",(0,t.jsx)(n.code,{children:"power_nominal_w"}),";"]}),"\n",(0,t.jsx)(n.li,{children:"inverts the power curve to select a concurrency cap;"}),"\n",(0,t.jsx)(n.li,{children:"recalibrates the analytical queue at that cap;"}),"\n",(0,t.jsx)(n.li,{children:"estimates P99 TTFT; and"}),"\n",(0,t.jsxs)(n.li,{children:["optionally runs DES with ",(0,t.jsx)(n.code,{children:"--verify-des"}),"."]}),"\n"]}
1),"\n",(0,t.jsx)(n.p,{children:"The requested percentage can be clipped by the idle-power floor, so inspect\nthe reported watts rather than assuming the target was reached exactly."}),"\n",(0,t.jsxs)(n.p,{children:["This command does not change ",(0,t.jsx)(n.code,{children:"max_num_seqs"}),", control a live vLLM server, or\ncommunicate with a power system. It produces a modeled trade-off curve for an\nexternal controller design."]}),"\n",(0,t.jsx)(n.h2,{id:"calibrate-a-profile",children:"Calibrate a profile"}),"\n",(0,t.jsx)(n.p,{children:"Calibrate performance before power; tokens per watt is not meaningful if the\nthroughput model is wrong."}),"\n",(0,t.jsxs)(n.ol,{children:["\n",(0,t.jsx)(n.li,{children:"Fix the model, precision, tensor parallelism, vLLM version, context mix,\nclocks, and power limit."}),"\n",(0,t.jsxs)(n.li,{children:["Measure TTFT, token throughput, active sequences, and KV capacity at several\nsteady-state loads. Fit ",(0,t.jsx)(n.code,{children:"W"}),", ",(0,t.jsx)(n.code,{children:"H"}),", ",(0,t.jsx)(n.code,{children:"calibration_ctx"}),", and ",(0,t.jsx)(n.code,{children:"max_slots"}),"."]}),"\n",(0,t.jsx)(n.li,{children:"Record board power from platform telemetry at the same loads, including low\nconcurrency and the highest sustainable operating point."}),"\n",(0,t.jsx)(n.li,{children:"Fit the linear or logistic curve to those points. Do not use TDP as a\nmeasured nominal value."}),"\n",(0,t.jsx)(n.li,{children:"Validate on held-out loads and a production-shaped token distribution."}),"\n",(0,t.jsx)(n.li,{children:"Store the measurement date and environment with the profile; remeasure\nafter material runtime or model changes."}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:"A source-defined manual profile can be constructed explicitly:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-python",children:'from fleet_sim.gpu_profiles.manual import ManualProfile\n\nprofile = ManualProfile(\n name="measured-model-on-target-gpu",\n W=0.006,\n H=0.0004,\n calibration_ctx=8192,\n chunk=512,\n blk_size=16,\n total_kv_blks=50000,\n max_slots=96,\n cost_per_hr=0.0,\n power_idle_w=180.0,\n power_nominal_w=360.0,\n power_logistic_k=0.9,\n power_logistic_x0=3.5,\n)\n'})}),"\n",(0,t.jsx)(n.p,{children:"The numbers above illustrate the required fields; they are not recommended\nvalues for any device."}),"\n",(0,t.jsx)(n.h2,{id:"reporting-checklist",children:"Reporting checklist"}),"\n",(0,t.jsx)(n.p,{children:"An energy result should state:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"whether watts mean GPU board power or whole-system/facility power;"}),"\n",(0,t.jsx)(n.li,{children:"model, dtype, parallel layout, runtime, clocks, and power limit;"}),"\n",(0,t.jsx)(n.li,{children:"workload and output-length distribution;"}),"\n",(0,t.jsx)(n.li,{children:"measurement source and fitted curve error;"}),"\n",(0,t.jsx)(n.li,{children:"fleet counts and utilization for every pool;"}),"\n",(0,t.jsx)(n.li,{children:"whether latency was analytical, DES, or load-test measured; and"}),"\n",(0,t.jsx)(n.li,{children:"uncertainty introduced by any unmeasured profile."}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:"Do not turn modeled board-power savings into facility energy, emissions, or\ndemand-response commitments without accounting for hosts, networking, storage,\ncooling, power conversion, workload displacement, and real controller\nbehavior."})]})}function h(e={}){const{wrapper:n}={...(0,l.R)(),...e.components};return n?(0,t.jsx)(n,{...e,children:(0,t.jsx)(c,{...e})}):c(e)}},28453(e,n,o){o.d(n,{R:()=>i,x:()=>s});var r=o(96540);const t={},l=r.createContext(t);function i(e){const n=r.useContext(l);return r.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function s(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:i(e.components),r.createElement(l.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.