https://cadenceworkflow.io/assets/js/3b0b63d3.1cec5530.js

1"use strict";(globalThis.webpackChunkcadence=globalThis.webpackChunkcadence||[]).push([[4500],{24526(e,i,n){n.r(i),n.d(i,{assets:()=>l,contentTitle:()=>o,default:()=>h,frontMatter:()=>r,metadata:()=>t,toc:()=>c});var t=n(16872),s=n(74848),a=n(28453);const r={title:"Adaptive Tasklist Scaler",description:"The Adaptive Tasklist Scaler is a new Cadence Matching service component that automatically adjusts tasklist partition counts based on live traffic, eliminating the need for manual configuration and reducing operational incidents.",keywords:["cadence adaptive tasklist scaler","cadence tasklist partitions","cadence matching service","cadence scalable tasklist","cadence auto scaling","cadence operational overhead","cadence partition configuration"],date:new Date("2025-06-30T00:00:00.000Z"),authors:"shaddoll",tags:["deep-dive","cadence-operations","cadence-matching"],subtitle:"test"},o=void 0,l={authorsImageUrls:[void 0]},c=[{value:"What is a Scalable Tasklist?",id:"what-is-a-scalable-tasklist",level:2},{value:"How Is the Number of Partitions Manually Configured?",id:"how-is-the-number-of-partitions-manually-configured",level:3},{value:"How Does Adaptive Tasklist Scaler Work?",id:"how-does-adaptive-tasklist-scaler-work",level:2},{value:"1. Migrating Configuration to the Database",id:"1-migrating-configuration-to-the-database",level:3},{value:"2. Monitoring Tasklist Traffic",id:"2-monitoring-tasklist-traffic",level:3},{value:"Enabling Adaptive Tasklist Scaler",id:"enabling-adaptive-tasklist-scaler",level:2},{value:"Prerequisites",id:"prerequisites",level:3},{value:"Configuration",id:"configuration",level:3},{value:"Monitoring and Metrics",id:"monitoring-and-metrics",level:2},{value:"QPS and Thresholds",id:"qps-and-thresholds",level:3},{value:"Partition Configurations",id:"partition-configurations",level:3},{value:"Status at Uber",id:"status-at-uber",level:2}];function d(e){const i={a:"a",blockquote:"blockquote",code:"code",h2:"h2",h3:"h3",hr:"hr",img:"img",li:"li",ol:"ol",p:"p",strong:"strong",ul:"ul",...(0,a.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(i.p,{children:"At Uber, we previously relied on a dynamic configuration service to manually control the number of partitions for scalable tasklists. This configuration approach introduced several operational challenges:"}),"\n",(0,s.jsxs)(i.ul,{children:["\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.strong,{children:"Error-prone:"})," Manual updates and deployments were required."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.strong,{children:"Unresponsive:"})," Adjustments were typically reactive, often triggered by customer reports or observed backlogs."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.strong,{children:"Irreversible:"})," Once increased, the number of partitions was rarely decreased due to the complexity of the two-phase process, especially when anticipating future traffic spikes."]}),"\n"]}),"\n",(0,s.jsxs)(i.p,{children:["To address these issues, we introduced a new component in the Cadence Matching service: ",(0,s.jsx)(i.strong,{children:"Adaptive Tasklist Scaler"}),". This component dynamically monitors tasklist traffic and adjusts partition counts automatically. Since its rollout, we've seen a significant reduction in incidents and operational overhead caused by misconfigured tasklists."]}),"\n",(0,s.jsx)(i.hr,{}),"\n",(0,s.jsx)(i.h2,{id:"what-is-a-scalable-tasklist",children:"What is a Scalable Tasklist?"}),"\n",(0,s.jsxs)(i.p,{children:["A ",(0,s.jsx)(i.strong,{children:"scalable tasklist"})," is one that supports multiple partitions. Since Cadence\u2019s Matching service is sharded by tasklist, all requests to a specific tasklist are routed to a single Matching host. To avoid bottlenecks and enhance scalability, tasklists can be partitioned so that multiple Matching hosts handle traffic concurrently."]}),"\n",(0,s.jsxs)(i.p,{children:["These partitions are transparent to clients. When a request arrives at the Cadence server for a scalable tasklist, the server selects an appropriate partition. More details can be found in ",(0,s.jsx)(i.a,{href:"https://github.com/cadence-workflow/cadence/blob/v1.3.1/docs/scalable_tasklist.md",children:"this document"}),"."]}),"\n",(0,s.jsx)(i.h3,{id:"how-is-the-number-of-partitions-manually-configured",children:"How Is the Number of Partitions Manually Configured?"}),"\n",(0,s.jsx)(i.p,{children:"The number of partitions for a tasklist is controlled by two dynamic configuration properties:"}),"\n",(0,s.jsxs)(i.ol,{children:["\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.a,{href:"https://github.com/cadence-workflow/cadence/blob/v1.2.13/common/dynamicconfig/constants.go#L3350",children:(0,s.jsx)(i.co

1de,{children:"matching.numTasklistReadPartitions"})}),": Specifies the number of ",(0,s.jsx)(i.strong,{children:"read"})," partitions."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.a,{href:"https://github.com/cadence-workflow/cadence/blob/v1.2.13/common/dynamicconfig/constants.go#L3344",children:(0,s.jsx)(i.code,{children:"matching.numTasklistWritePartitions"})}),": Specifies the number of ",(0,s.jsx)(i.strong,{children:"write"})," partitions."]}),"\n"]}),"\n",(0,s.jsxs)(i.p,{children:["To prevent misconfiguration, a guardrail is in place to ensure that the number of read partitions is ",(0,s.jsx)(i.strong,{children:"never less than"})," the number of write partitions."]}),"\n",(0,s.jsxs)(i.p,{children:["When ",(0,s.jsx)(i.strong,{children:"increasing"})," the number of partitions, both properties are typically updated simultaneously. However, due to the guardrail, the order of updates doesn't matter\u2014read and write partitions can be increased in any sequence."]}),"\n",(0,s.jsxs)(i.p,{children:["In contrast, ",(0,s.jsx)(i.strong,{children:"decreasing"})," the number of partitions is more complex and requires a ",(0,s.jsx)(i.strong,{children:"two-phase process"}),":"]}),"\n",(0,s.jsxs)(i.ol,{children:["\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.strong,{children:"First"}),", reduce the number of write partitions."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.strong,{children:"Then"}),", wait for any backlog in the decommissioned partitions to drain completely."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.strong,{children:"Finally"}),", reduce the number of read partitions."]}),"\n"]}),"\n",(0,s.jsx)(i.p,{children:"Because this process is tedious, error-prone, and backlog-sensitive, it is rarely performed in production environments."}),"\n",(0,s.jsx)(i.hr,{}),"\n",(0,s.jsx)(i.h2,{id:"how-does-adaptive-tasklist-scaler-work",children:"How Does Adaptive Tasklist Scaler Work?"}),"\n",(0,s.jsx)(i.p,{children:"The architecture of the adaptive tasklist scaler is shown below:"}),"\n",(0,s.jsx)(i.p,{children:(0,s.jsx)(i.img,{alt:"adaptive tasklist scaler architecture",src:n(14126).A+"",width:"1200",height:"980"})}),"\n",(0,s.jsx)(i.h3,{id:"1-migrating-configuration-to-the-database",children:"1. Migrating Configuration to the Database"}),"\n",(0,s.jsx)(i.p,{children:"The first key change was migrating partition count configuration from dynamic config to the Cadence cluster\u2019s database. This allows the configuration to be updated programmatically."}),"\n",(0,s.jsxs)(i.ul,{children:["\n",(0,s.jsxs)(i.li,{children:["The ",(0,s.jsx)(i.strong,{children:"adaptive tasklist scaler"})," runs in the root partition only."]}),"\n",(0,s.jsx)(i.li,{children:"It reads and updates the partition count."}),"\n",(0,s.jsxs)(i.li,{children:["Updates propagate to non-root partitions via a ",(0,s.jsx)(i.strong,{children:"push model"}),", and to pollers and producers via a ",(0,s.jsx)(i.strong,{children:"pull model"}),"."]}),"\n",(0,s.jsxs)(i.li,{children:["A ",(0,s.jsx)(i.strong,{children:"version number"})," is associated with each config. The version only increments through scaler updates, ensuring monotonicity and consistency across components."]}),"\n"]}),"\n",(0,s.jsx)(i.h3,{id:"2-monitoring-tasklist-traffic",children:"2. Monitoring Tasklist Traffic"}

1),"\n",(0,s.jsxs)(i.p,{children:["The scaler periodically monitors the ",(0,s.jsx)(i.strong,{children:"write QPS"})," of each tasklist."]}),"\n",(0,s.jsxs)(i.ul,{children:["\n",(0,s.jsxs)(i.li,{children:["If QPS exceeds an upscale threshold for a sustained period, the number of ",(0,s.jsx)(i.strong,{children:"read and write partitions"})," is increased proportionally."]}),"\n",(0,s.jsxs)(i.li,{children:["If QPS falls below a downscale threshold, only the ",(0,s.jsx)(i.strong,{children:"write partitions"})," are reduced initially. The system then waits for drained partitions to clear before reducing the number of ",(0,s.jsx)(i.strong,{children:"read partitions"}),", ensuring backlog-free downscaling."]}),"\n"]}),"\n",(0,s.jsx)(i.hr,{}),"\n",(0,s.jsx)(i.h2,{id:"enabling-adaptive-tasklist-scaler",children:"Enabling Adaptive Tasklist Scaler"}),"\n",(0,s.jsx)(i.h3,{id:"prerequisites",children:"Prerequisites"}),"\n",(0,s.jsxs)(i.p,{children:["To use this feature, upgrade Cadence to ",(0,s.jsx)(i.a,{href:"https://github.com/cadence-workflow/cadence/tree/v1.3.0",children:"v1.3.0 or later"}),"."]}),"\n",(0,s.jsxs)(i.p,{children:["Also, migrate tasklist partition configurations to the database using ",(0,s.jsx)(i.a,{href:"https://github.com/cadence-workflow/cadence/blob/v1.3.0/docs/migration/tasklist-partition-config.md",children:"this guide"}),"."]}),"\n",(0,s.jsx)(i.h3,{id:"configuration",children:"Configuration"}),"\n",(0,s.jsx)(i.p,{children:"The scaler is governed by the following dynamic configuration parameters:"}),"\n",(0,s.jsxs)(i.ul,{children:["\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"matching.enableAdaptiveScaler"}),": Enables the scaler at the tasklist level."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"matching.partitionUpscaleSustainedDuration"}),": Duration that QPS must stay above threshold before triggering upscale."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"matching.partitionDownscaleSustainedDuration"}),": Duration below threshold required before triggering downscale."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"matching.adaptiveScalerUpdateInterval"}),": Frequency at which the scaler evaluates and updates partition counts."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"matching.partitionUpscaleRPS"}),": QPS threshold per partition that triggers upscale."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"matching.partitionDownscaleFactor"}),": Factor applied to introduce hysteresis, lowering the QPS threshold for downscaling to avoid oscillations."]}),"\n"]}),"\n",(0,s.jsx)(i.hr,{}),"\n",(0,s.jsx)(i.h2,{id:"monitoring-and-metrics",children:"Monitoring and Metrics"}),"\n",(0,s.jsx)(i.p,{children:"Several metrics have been introduced to help monitor the scaler\u2019s behavior:"}),"\n",(0,s.jsx)(i.h3,{id:"qps-and-thresholds",children:"QPS and Thresholds"}),"\n",(0,s.jsxs)(i.ul,{children:["\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"estimated_add_task_qps_per_tl"}),": Estimated QPS of task additions per tasklist."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"tasklist_partition_upscale_threshold"}),": Upscale threshold for task additions."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"tasklist_partition_downscale_threshold"}),": Downscale threshold for task additions."]}),"\n"]}),"\n",(0,s.jsxs)(i.blockquote,{children:["\n",(0,s.jsxs)(i.p,{children:["The ",(0,s.jsx)(i.code,{children:"estimated_add_task_qps_per_tl"})," value should remain between the upscale and downscale thresholds. If not, the scaler may not be functioning properly."]}),"\n"]}),"\n",(0,s.jsx)(i.h3,{id:"partition-configurations",children:"Partition Configurations"}),"\n",(0,s.jsxs)(i.ul,{children:["\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"task_list_partition_config_num_read"}),": Number of current ",(0,s.jsx)(i.strong,{children:"read"})," partitions."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"task_list_partition_config_num_write"}),": Number of current ",(0,s.jsx)(i.strong,{children:"write"})," partitions."]}

1),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.code,{children:"task_list_partition_config_version"}),": Version of the current partition configuration."]}),"\n"]}),"\n",(0,s.jsx)(i.p,{children:"These metrics are emitted by various components: root and non-root partitions, pollers, and producers. Their values should align under normal conditions, except immediately after updates."}),"\n",(0,s.jsx)(i.hr,{}),"\n",(0,s.jsx)(i.h2,{id:"status-at-uber",children:"Status at Uber"}),"\n",(0,s.jsxs)(i.p,{children:["We enabled adaptive tasklist scaler across all Uber clusters in ",(0,s.jsx)(i.strong,{children:"March 2025"}),". Since its deployment:"]}),"\n",(0,s.jsxs)(i.ul,{children:["\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.strong,{children:"Zero incidents"})," have been reported due to misconfigured tasklists."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.strong,{children:"Operational workload"})," related to manual scaling has been eliminated."]}),"\n",(0,s.jsxs)(i.li,{children:[(0,s.jsx)(i.strong,{children:"Scalability and resilience"})," of Matching service have improved significantly."]}),"\n"]})]})}function h(e={}){const{wrapper:i}={...(0,a.R)(),...e.components};return i?(0,s.jsx)(i,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},14126(e,i,n){n.d(i,{A:()=>t});const t=n.p+"assets/images/architecture-0c083477bee0b8657dc4cea752dc34ba.png"},28453(e,i,n){n.d(i,{R:()=>r,x:()=>o});var t=n(96540);const s={},a=t.createContext(s);function r(e){const i=t.useContext(a);return t.useMemo(function(){return"function"==typeof e?e(i):{...i,...e}},[i,e])}function o(e){let i;return i=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:r(e.components),t.createElement(a.Provider,{value:i},e.children)}},16872(e){e.exports=JSON.parse('{"permalink":"/blog/2025/06/30/adaptive-tasklist-scaler","editUrl":"https://github.com/cadence-workflow/Cadence-Docs/tree/master/blog/2025-06-30-adaptive-tasklist-scaler.md","source":"@site/blog/2025-06-30-adaptive-tasklist-scaler.md","title":"Adaptive Tasklist Scaler","description":"The Adaptive Tasklist Scaler is a new Cadence Matching service component that automatically adjusts tasklist partition counts based on live traffic, eliminating the need for manual configuration and reducing operational incidents.","date":"2025-06-30T00:00:00.000Z","tags":[{"inline":false,"label":"Deep Dives","permalink":"/blog/tags/deep-dives","description":"Deep Dives tag description"},{"inline":false,"label":"Cadence Operations","permalink":"/blog/tags/cadence-operations","description":"Cadence Operations tag description"},{"inline":false,"label":"Cadence Matching","permalink":"/blog/tags/cadence-matching","description":"Cadence Matching is one of the core components responsible for matching tasks to workers."}],"readingTime":4.2,"hasTruncateMarker":true,"authors":[{"name":"Zijian Chen","title":"Software Engineer @ Uber","url":"https://www.linkedin.com/in/zijian-chen-5868938b/","page":{"permalink":"/blog/authors/shaddoll"},"socials":{"linkedin":"https://www.linkedin.com/in/zijian-chen-5868938b/","github":"https://github.com/Shaddoll"},"imageURL":"https://github.com/Shaddoll.png","key":"shaddoll"}],"frontMatter":{"title":"Adaptive Tasklist Scaler","description":"The Adaptive Tasklist Scaler is a new Cadence Matching service component that automatically adjusts tasklist partition counts based on live traffic, eliminating the need for manual configuration and reducing operational incidents.","keywords":["cadence adaptive tasklist scaler","cadence tasklist partitions","cadence matching service","cadence scalable tasklist","cadence auto scaling","cadence operational overhead","cadence partition configuration"],"date":"2025-06-30T00:00:00.000Z","authors":"shaddoll","tags":["deep-dive","cadence-operations","cadence-matching"],"subtitle":"test"},"unlisted":false,"prevItem":{"title":"Safe deployments of Versioned workflows","permalink":"/blog/2025/07/01/2025-07-01-safe-deployment-of-versioned-workflows/safe-deployment-of-versioned-workflows"},"nextItem":{"title":"Introducing cadence-web v4.0.0","permalink":"/blog/2025/04/11/2025-04-11-announcing-cadence-web-v4/announcing-cadence-web-v4"}}')}}]);