https://conduitio.github.io/assets/js/8e0fa5f2.aed078ff.js

1"use strict";(self.webpackChunkconduit_site=self.webpackChunkconduit_site||[]).push([[850],{6084:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>s,default:()=>u,frontMatter:()=>a,metadata:()=>o,toc:()=>d});const o=JSON.parse('{"id":"scaling/conduit-operator","title":"Conduit Operator","description":"The Conduit Operator is a Kubernetes operator designed to simplify the management and orchestration of Conduit instances. The operator extends the Kubernetes API with a custom resource for Conduit.","source":"@site/docs/3-scaling/0-conduit-operator.mdx","sourceDirName":"3-scaling","slug":"/scaling/conduit-operator","permalink":"/docs/scaling/conduit-operator","draft":false,"unlisted":false,"editUrl":"https://github.com/conduitio/conduitio.github.io/edit/main/docs/3-scaling/0-conduit-operator.mdx","tags":[],"version":"current","lastUpdatedAt":1751451781000,"sidebarPosition":0,"frontMatter":{"title":"Conduit Operator","toc_max_heading_level":6},"sidebar":"docsSidebar","previous":{"title":"How it works","permalink":"/docs/developing/processors/how-it-works"},"next":{"title":"Roadmap","permalink":"/docs/future/roadmap"}}');var i=t(74848),r=t(28453);const a={title:"Conduit Operator",toc_max_heading_level:6},s=void 0,c={},d=[{value:"Overview",id:"overview",level:3},{value:"Quickstart",id:"quickstart",level:3},{value:"Installing standalone connectors",id:"installing-standalone-connectors",level:3},{value:"<code>plugin</code> and <code>pluginVersion</code>",id:"plugin-and-pluginversion",level:4},{value:"Example",id:"example",level:4},{value:"Schema Support",id:"schema-support",level:3},{value:"Example",id:"example-1",level:4},{value:"Deploying",id:"deploying",level:3},{value:"Using the helm chart repository",id:"using-the-helm-chart-repository",level:3}];function l(e){const n={a:"a",admonition:"admonition",code:"code",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",ul:"ul",...(0,r.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsxs)(n.p,{children:["The Conduit Operator is a ",(0,i.jsx)(n.a,{href:"https://kubernetes.io/docs/concepts/extend-kubernetes/operator/",children:"Kubernetes operator"})," designed to simplify the management and orchestration of Conduit instances. The operator extends the ",(0,i.jsx)(n.a,{href:"https://kubernetes.io/docs/concepts/overview/kubernetes-api/",children:"Kubernetes API"})," with a ",(0,i.jsx)(n.a,{href:"https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/",children:"custom resource"})," for Conduit."]}),"\n",(0,i.jsx)(n.p,{children:"This custom resource allows users to define and manage Conduit instances declaratively, enabling seamless integration with Kubernetes' native features such as scaling, monitoring, and logging."}),"\n",(0,i.jsx)(n.h3,{id:"overview",children:"Overview"}),"\n",(0,i.jsx)(n.p,{children:"Conduit pipelines are represented as Conduit custom resources, where each pipeline will be provisioned\nas a distinct Conduit instance with its own lifecycle."}),"\n",(0,i.jsxs)(n.p,{children:["The Conduit custom resource definition format is very similar to that of ",(0,i.jsx)(n.a,{href:"/docs/using/pipelines/configuration-file",children:"pipeline configurations"}),":"]}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-yaml",children:'apiVersion: operator.conduitio.github.io/v1alpha\nkind: Conduit\nmetadata:\n  name: conduit-generator\nspec:\n  running: true\n  name: generator.log\n  description: generator pipeline\n  connectors:\n    - name: source-connector\n      type: source\n      plugin: builtin:generator\n      settings:\n        - name: format.type\n          value: structured\n        - name: format.options.id\n          value: "int"\n        - name: format.options.name\n          value: "string"\n        - name: format.options.company\n          value: "string"\n        - name: format.options.trial\n          value: "bool"\n        - name: recordCount\n          value: "3"\n    - name: destination-connector\n      type: destination\n      plugin: builtin:log\n      \n'})}

1),"\n",(0,i.jsx)(n.h3,{id:"quickstart",children:"Quickstart"}),"\n",(0,i.jsx)(n.admonition,{type:"info",children:(0,i.jsxs)(n.p,{children:["To get started with the Conduit Operator, follow the steps described on the ",(0,i.jsx)(n.a,{href:"https://github.com/ConduitIO/conduit-operator?tab=readme-ov-file#quickstart",children:"Conduit Operator GitHub repository"}),"."]})}),"\n",(0,i.jsx)(n.h3,{id:"installing-standalone-connectors",children:"Installing standalone connectors"}),"\n",(0,i.jsxs)(n.h4,{id:"plugin-and-pluginversion",children:[(0,i.jsx)(n.code,{children:"plugin"})," and ",(0,i.jsx)(n.code,{children:"pluginVersion"})]}),"\n",(0,i.jsxs)(n.p,{children:["A standalone connector can be installed by providing the connector name via ",(0,i.jsx)(n.code,{children:"plugin"}),", and version via ",(0,i.jsx)(n.code,{children:"pluginVersion"}),"."]}),"\n",(0,i.jsxs)(n.p,{children:["The operator will then install the connector and create the necessary resources to run it. These connectors need to be referred to by their GitHub ",(0,i.jsx)(n.code,{children:"organization/repository"})," names and are currently limited to the ",(0,i.jsx)(n.a,{href:"https://github.com/ConduitIO/conduit-operator/blob/7321042d1fa1205f3c1e5faf70775184ebafedba/pkg/conduit/plugin.go#L9-L11",children:"following organizations"}),":"]}),"\n",(0,i.jsxs)(n.ul,{children:["\n",(0,i.jsx)(n.li,{children:(0,i.jsx)(n.a,{href:"https://github.com/conduitio",children:"conduitio"})}),"\n",(0,i.jsx)(n.li,{children:(0,i.jsx)(n.a,{href:"https://github.com/conduitio-labs",children:"conduitio-labs"})}),"\n",(0,i.jsx)(n.li,{children:(0,i.jsx)(n.a,{href:"https://github.com/meroxa",children:"meroxa"})}),"\n"]}),"\n",(0,i.jsx)(n.admonition,{type:"tip",children:(0,i.jsx)(n.p,{children:"Plugin version can optionally be specified or the latest will be used."})}),"\n",(0,i.jsx)(n.h4,{id:"example",children:"Example"}),"\n",(0,i.jsxs)(n.p,{children:["Here's a full example using ",(0,i.jsx)(n.a,{href:"https://github.com/conduitio/conduit-connector-generator",children:(0,i.jsx)(n.code,{children:"conduitio/conduit-connector-generator"})})," as ",(0,i.jsx)(n.code,{children:"plugin"})," and ",(0,i.jsx)(n.code,{children:"v0.8.0"})," as ",(0,i.jsx)(n.code,{children:"pluginVersion"}),":"]}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-yaml",children:'apiVersion: operator.conduitio.github.io/v1alpha\nkind: Conduit\nmetadata:\n  name: conduit-generator\nspec:\n  running: true\n  name: generator.log\n  description: generator pipeline\n  connectors:\n    - name: source-connector\n      type: source\n      plugin: conduitio/conduit-connector-generator\n      pluginVersion: v0.8.0\n      settings:\n        - name: format.type\n          value: structured\n        - name: format.options.id\n          value: "int"\n        - name: format.options.name\n          value: "string"\n        - name: format.options.company\n          value: "string"\n        - name: format.options.trial\n          value: "bool"\n        - name: recordCount\n          value: "3"\n    - name: destination-connector\n      type: destination\n      plugin: conduitio/conduit-connector-log\n'})}),"\n",(0,i.jsx)(n.h3,{id:"schema-support",children:"Schema Support"}),"\n",(0,i.jsxs)(n.p,{children:["As of ",(0,i.jsx)(n.a,{href:"/changelog/2024-08-19-conduit-0-11-0-release",children:"v0.11.0"})," Conduit supports the use of a schema registry."]}),"\n",(0,i.jsxs)(n.p,{children:["This allows connectors to automatically extract or use the schema referred to by the ",(0,i.jsx)(n.a,{href:"/docs/using/opencdc-record",children:"OpenCDC record"})," to encode/decode data."]}),"\n",(0,i.jsxs)(n.p,{children:["By default Conduit uses a builtin schema registry, however in certain use cases a schema registry needs to be shared between multiple instances. The Conduit resource allows for schema registry to be defined as of ",(0,i.jsx)(n.a,{href:"https://github.com/ConduitIO/conduit-operator/releases/tag/v0.0.2",children:"v0.2.0"}),"."]}),"\n",(0,i.jsxs)(n.p,{children:["Details are provided directly via the Conduit resource. ",(0,i.jsx)(n.code,{children:"basicAuthUser"})," and ",(0,i.jsx)(n.code,{children:"basicAuthPassword"})," can be provided as secrets but will be copied over into controller-owned secrets for management and ease of use."]}),"\n",(0,i.jsx)(n.h4,{id:"example-1",children:"Example"}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-yaml",children:'apiVersion: operator.conduitio.github.io/v1alpha\nkind: Conduit\nmetadata:\n  name: conduit-generator-schema-registry\nspec:\n  running: true\n  name: generator.standalone.log\n  description: generator pipeline\n  schemaRegistry:\n    url: http://apicurio:8080/apis/ccompat/v7\n    # basicAuthUser:\n    #   - value: <schemaUser>\n    # basicAuthPassword:\n    #   - secretRef:\n    #     key: schema-registry-password\n    #     name: schema-registry-secret\n  connectors:\n    - name: source-connector\n      type: source\n      plugin: conduitio/conduit-connector-generator\n      settings:\n        - name: format.type\n          value: structured\n        - name: format.options.id\n          value: "int"\n        - name: format.options.name\n          value: "string"\n        - name: format.options.company\n          value: "string"\n        - name: format.options.trial\n          value: "bool"\n        - name: recordCount\n          value: "3"\n    - name: destination-connector\n      type: destination\n      plugin: conduitio/conduit-connector-log\n      pluginVersion: v0.4.0\n'})}),"\n",(0,i.jsx)(n.h3,{id:"deploying",children:"Deploying"}),"\n",(0,i.jsxs)(n.p,{children:["The operator provides a ",(0,i.jsx)(n.a,{href:"https://github.com/ConduitIO/conduit-operator/blob/main/charts/conduit-operator",children:"helm chart"})," that can be used for deployment on any cluster. Additional metadata can be injected into each provisioned conduit instance via the ",(0,i.jsx)(n.code,{children:"controller.conduitMetadata"})," configuration."]}),"\n",(0,i.jsx)(n.p,{children:"For example, to instruct a prometheus instance to scrape each Conduit instance metrics:"}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-yaml",children:"controller:\n  conduitMetadata:\n    podAnnotations:\n      prometheus.io/scrape: true\n      prometheus.io/path: /metrics\n      prometheus.io/port: 8080\n"})}),"\n",(0,i.jsxs)(n.p,{children:["For more configuration options see ",(0,i.jsx)(n.a,{href:"https://github.com/ConduitIO/conduit-operator/blob/main/charts/conduit-operator/values.yaml",children:"charts/conduit-operator/values.yaml"}),"."]}),"\n",(0,i.jsxs)(n.p,{children:["Additional metadata can be injected in each provisioned conduit instance via ",(0,i.jsx)(n.code,{children:"controller.conduitMetadata"})," configuration."]}),"\n",(0,i.jsx)(n.p,{children:"For example, to instruct a prometheus instance to scrape each Conduit instance metrics:"}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-yaml",children:"controller:\n  conduitMetadata:\n    podAnnotations:\n      prometheus.io/scrape: true\n      prometheus.io/path: /metrics\n      prometheus.io/port: 8080\n"})}),"\n",(0,i.jsxs)(n.p,{children:["For more configuration options see ",(0,i.jsx)(n.a,{href:"https://github.com/ConduitIO/conduit-operator/blob/main/charts/conduit-operator/values.yaml",children:"charts/conduit-operator/values.yaml"}),"."]}),"\n",(0,i.jsx)(n.h3,{id:"using-the-helm-chart-repository",children:"Using the helm chart repository"}),"\n",(0,i.jsxs)(n.p,{children:["Alternatively the operator can be deployed via the ",(0,i.jsx)(n.a,{href:"https://helm.sh/docs/topics/chart_repository/",children:"Helm repository"}),". To add the repository to your helm repos:"]}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-shell",children:"helm repo add conduit https://conduitio.github.io/conduit-operator\n"})}),"\n",(0,i.jsxs)(n.p,{children:["Install the operator in the ",(0,i.jsx)(n.code,{children:"conduit-operator"})," namespace of your cluster:"]}

1),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-shell",children:"helm install conduit-operator \\\n    conduit/conduit-operator --create-namespace -n conduit-operator\n"})})]})}function u(e={}){const{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,i.jsx)(n,{...e,children:(0,i.jsx)(l,{...e})}):l(e)}},28453:(e,n,t)=>{t.d(n,{R:()=>a,x:()=>s});var o=t(96540);const i={},r=o.createContext(i);function a(e){const n=o.useContext(r);return o.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function s(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:a(e.components),o.createElement(r.Provider,{value:n},e.children)}}}]);