1"use strict";(self.webpackChunkconduit_site=self.webpackChunkconduit_site||[]).push([[4688],{12542:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>a,default:()=>u,frontMatter:()=>s,metadata:()=>i,toc:()=>d});const i=JSON.parse('{"id":"using/connectors/configuration-parameters/rate-limiting","title":"Rate Limiting","description":"Destination connectors can be configured to limit the rate at which records can be written. This is especially useful when the destination resource has a rate limit to ensure that the connector does not exceed it. By default, Conduit does not limit the rate at which records are written.","source":"@site/docs/1-using/5-connectors/6-configuration-parameters/1-rate-limiting.mdx","sourceDirName":"1-using/5-connectors/6-configuration-parameters","slug":"/using/connectors/configuration-parameters/rate-limiting","permalink":"/docs/using/connectors/configuration-parameters/rate-limiting","draft":false,"unlisted":false,"editUrl":"https://github.com/conduitio/conduitio.github.io/edit/main/docs/1-using/5-connectors/6-configuration-parameters/1-rate-limiting.mdx","tags":[],"version":"current","lastUpdatedAt":1751447712000,"sidebarPosition":1,"frontMatter":{"title":"Rate Limiting"},"sidebar":"docsSidebar","previous":{"title":"Output Format","permalink":"/docs/using/connectors/configuration-parameters/output-format"},"next":{"title":"Schema Extraction","permalink":"/docs/using/connectors/configuration-parameters/schema-extraction"}}');var r=n(74848),o=n(28453);const s={title:"Rate Limiting"},a=void 0,c={},d=[{value:"Configuration parameters",id:"configuration-parameters",level:2},{value:"Example",id:"example",level:2}];function l(e){const t={code:"code",h2:"h2",li:"li",p:"p",pre:"pre",ul:"ul",...(0,o.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsx)(t.p,{children:"Destination connectors can be configured to limit the rate at which records can be written. This is especially useful when the destination resource has a rate limit to ensure that the connector does not exceed it. By default, Conduit does not limit the rate at which records are written."}),"\n",(0,r.jsx)(t.h2,{id:"configuration-parameters",children:"Configuration parameters"}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsxs)(t.li,{children:[(0,r.jsx)(t.code,{children:"sdk.rate.perSecond"}),": Maximum number of records written per second (0 means no rate limit)."]}),"\n",(0,r.jsxs)(t.li,{children:[(0,r.jsx)(t.code,{children:"sdk.rate.burst"}),": Allow bursts of at most X records (0 or less means that bursts are not limited). Only takes effect if a rate limit per second is set. Note that if ",(0,r.jsx)(t.code,{children:"sdk.batch.size"})," is bigger than ",(0,r.jsx)(t.code,{children:"sdk.rate.burst"}),", the effective batch size will be equal to ",(0,r.jsx)(t.code,{children:"sdk.rate.burst"}),"."]}),"\n"]}),"\n",(0,r.jsx)(t.h2,{id:"example",children:"Example"}),"\n",(0,r.jsxs)(t.p,{children:["The pipeline will generate structured records as fast as possible, and write them to the log. When you run it, you'll notice that by using both ",(0,r.jsx)(t.code,{children:"sdk.rate.perSecond"})," and ",(0,r.jsx)(t.code,{children:"sdk.rate.burst"}),", the log destination connector will limit the rate of processed records 8 records per second in batches of 2 (i.e. 2 records every 250ms)."]}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-yaml",children:"version: 2.2\npipelines:\n - id: generator-to-log\n status: running\n description: >\n Example pipeline using the generator source connector and the log destination connector.\n Showing how to limit the rate of processing records.\n connectors:\n - id: example\n type: source\n plugin: generator\n settings:\n rate: 0 # generating source records as fast as possible\n format.type: structured\n format.options.id: int\n format.options.name: string\n operations: create\n - id: log\n type: destination\n plugin: log\n settings:\n level: info\n\n # limit the rate of proccessing records\n sdk.rate.perSecond: 8\n sdk.rate.burst: 2\n"})})]})}function u(e={}){const{wrapper:t}={...(0,o.R)(),...e.components};return t?(0,r.jsx)(t,{...e,children:(0,r.jsx)(l,{...e})}):l(e)}},28453:(e,t,n)=>{n.d(t,{R:()=>s,x:()=>a});var i=n(96540);const r={},o=i.createContext(r);function s(e){const t=i.useContext(o);return i.useMemo(function(){return"function"==typeof e?e(t):{...t,...e}},[t,e])}function a(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:s(e.components),i.createElement(o.Provider,{value:t},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.