1"use strict";(self.webpackChunkfluvio_docs=self.webpackChunkfluvio_docs||[]).push([[5068],{22279:(e,r,n)=>{n.r(r),n.d(r,{assets:()=>c,contentTitle:()=>s,default:()=>h,frontMatter:()=>i,metadata:()=>l,toc:()=>a});var o=n(74848),t=n(28453);const i={sidebar_position:80,title:"Mirroring - Cluster to Cluster",description:"Mirroring fluvio topics between two local clusters"},s="Mirroring Two Local Clusters",l={id:"fluvio/tutorials/mirroring-two-clusters",title:"Mirroring - Cluster to Cluster",description:"Mirroring fluvio topics between two local clusters",source:"@site/versioned_docs/version-0.15.0/fluvio/tutorials/mirroring-two-clusters.mdx",sourceDirName:"fluvio/tutorials",slug:"/fluvio/tutorials/mirroring-two-clusters",permalink:"/docs/0.15.0/fluvio/tutorials/mirroring-two-clusters",draft:!1,unlisted:!1,editUrl:"https://github.com/infinyon/fluvio-docs/tree/main/versioned_docs/version-0.15.0/fluvio/tutorials/mirroring-two-clusters.mdx",tags:[],version:"0.15.0",lastUpdatedBy:"Felipe Cardozo",lastUpdatedAt:1752028303e3,sidebarPosition:80,frontMatter:{sidebar_position:80,title:"Mirroring - Cluster to Cluster",description:"Mirroring fluvio topics between two local clusters"},sidebar:"fluvio",previous:{title:"Mirroring - Raspberry Pi to Cluster",permalink:"/docs/0.15.0/fluvio/tutorials/mirroring-iot-local"},next:{title:"Overview",permalink:"/docs/0.15.0/fluvio/concepts/architecture/overview"}},c={},a=[{value:"Home and Remote",id:"home-and-remote",level:2},{value:"Dockerfile and Docker-compose",id:"dockerfile-and-docker-compose",level:2},{value:"Register Remote clusters on the Home",id:"register-remote-clusters-on-the-home",level:3},{value:"Create the mirror topic",id:"create-the-mirror-topic",level:3},{value:"Generate Metadata for Remote Clusters",id:"generate-metadata-for-remote-clusters",level:3},{value:"Producing and Consuming on Mirroring",id:"producing-and-consuming-on-mirroring",level:2},{value:"Produce to remote cluster",id:"produce-to-remote-cluster",level:3},{value:"Consume from Home cluster",id:"consume-from-home-cluster",level:3},{value:"Related",id:"related",level:4}];function d(e){const r={a:"a",code:"code",h1:"h1",h2:"h2",h3:"h3",h4:"h4",header:"header",hr:"hr",li:"li",p:"p",pre:"pre",ul:"ul",...(0,t.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(r.header,{children:(0,o.jsx)(r.h1,{id:"mirroring-two-local-clusters",children:"Mirroring Two Local Clusters"})}),"\n",(0,o.jsx)(r.p,{children:"This tutorial is a demonstration of mirroring between two clusters running on Docker."}),"\n",(0,o.jsx)(r.h2,{id:"home-and-remote",children:"Home and Remote"}),"\n",(0,o.jsx)(r.p,{children:"To understand mirroring, we need to understand what is a Home and a Remote cluster:"}),"\n",(0,o.jsxs)(r.ul,{children:["\n",(0,o.jsx)(r.li,{children:"Home cluster is the target cluster that will receive and consume data."}),"\n",(0,o.jsx)(r.li,{children:"Remote cluster is the source cluster that will send data."}),"\n"]}),"\n",(0,o.jsx)(r.h2,{id:"dockerfile-and-docker-compose",children:"Dockerfile and Docker-compose"}),"\n",(0,o.jsx)(r.p,{children:"Copy the docker-compose.yaml and Dockerfile to a directory."}),"\n",(0,o.jsx)(r.p,{children:"docker-compose.yaml:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-yaml",children:'version: "3"\nservices:\n ## Home Cluster\n sc-home:\n build:\n context: .\n dockerfile: Dockerfile\n container_name: sc-home\n environment:\n - RUST_LOG=info\n entrypoint: >\n /bin/sh -c "\n fluvio profile add docker 0.0.0.0:9003 docker;\n bash -c \'sleep 5 ; fluvio cluster spu register --id 5001 -p spu-home:9010 --private-server spu-home:9011\' &\n ./fluvio-run sc --local /fluvio/metadata\n "\n volumes:\n - ./home/fluvio-metadata:/fluvio/metadata\n - ./shared:/shared\n spu-home:\n build:\n context: .\n dockerfile: Dockerfile\n container_name: spu-home\n hostname: spu-home\n volumes:\n - ./home/fluvio-data:/fluvio/data\n environment:\n - RUST_LOG=info\n command: "./fluvio-run spu -i 5001 -p spu-home:9010 -v spu-home:9011 --sc-addr sc-home:9004 --log-base-dir /fluvio/data"\n depends_on:\n - sc-home\n ## Remote Cluster \n sc-remote:\n build:\n context: .\n dockerfile: Dockerfile\n container_name: sc-remote\n hostname: sc-remote\n environment:\n - RUST_LOG=info\n entrypoint: >\n /bin/sh -c "\n fluvio profile add docker 0.0.0.0:9003 docker;\n bash -c \'sleep 5 ; fluvio cluster spu register --id 5001 -p spu-remote:9010 --private-server spu-remote:9011\' &\n ./fluvio-run sc --local /fluvio/metadata\n "\n volumes:\n - ./remote/fluvio-metadata:/fluvio/metadata\n - ./shared:/shared\n spu-remote:\n build:\n context: .\n dockerfile: Dockerfile\n container_name: spu-remote\n hostname: spu-remote\n volumes:\n - ./remote/fluvio-data:/fluvio/data\n environment:\n - RUST_LOG=info\n command: "./fluvio-run spu -i 5001 -p spu-remote:9010 -v spu-remote:9011 --sc-addr sc-remote:9004 --log-base-dir /fluvio/data"\n depends_on:\n - sc-remote\n'})}),"\n",(0,o.jsx)(r.p,{children:"Dockerfile:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-dockerfile",children:"FROM infinyon/fluvio:latest\n\nRUN apk update\nRUN apk add curl unzip bash\nRUN curl -fsS https://hub.infinyon.cloud/install/install.sh?ctx=dc | VERSION='latest' bash\n\nRUN mv /root/.fluvio/bin/fluvio /usr/local/bin/fluvio\n"})}),"\n",(0,o.jsx)(r.p,{children:"Start Docker Containers with:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"docker-compose up -d\n"})}),"\n",(0,o.jsx)(r.h3,{id:"register-remote-clusters-on-the-home",children:"Register Remote cluster
1s on the Home"}),"\n",(0,o.jsx)(r.p,{children:"First, enter into the home cluster shell:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"docker exec -it sc-home /bin/sh\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Use the ",(0,o.jsx)(r.code,{children:"remote"})," CLI to register the remote clusters with the home cluster:"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio remote register docker-remote\n"})}),"\n",(0,o.jsx)(r.p,{children:"List remote clusters to check their status:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio remote list\n"})}),"\n",(0,o.jsx)(r.p,{children:"It should show the following:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:" REMOTE SC STATUS SPU STATUS LAST SEEN ERRORS\n docker-remote Waiting Waiting - -\n"})}),"\n",(0,o.jsx)(r.h3,{id:"create-the-mirror-topic",children:"Create the mirror topic"}),"\n",(0,o.jsxs)(r.p,{children:["Mirror topics on the home clusters has multiple partitions, where each partition has a ",(0,o.jsx)(r.code,{children:"1-to-1"})," relationship with the remote cluster."]}),"\n",(0,o.jsx)(r.p,{children:"Create the mirror topic, then add the remote to it:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio topic create mirror-topic --mirror\nfluvio topic add-mirror mirror-topic docker-remote\n"})}),"\n",(0,o.jsx)(r.p,{children:"or apply a json file with an array of remotes that you want to assign when creating a topic:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"echo '[\"docker-remote\"]' > assignment_file.json\nfluvio topic create mirror-topic --mirror-apply assignment_file.json\n"})}),"\n",(0,o.jsx)(r.p,{children:"List partitions to check the assignment:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio partition list\n"})}),"\n",(0,o.jsx)(r.p,{children:"It should display the partition that we created:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:" TOPIC PARTITION LEADER MIRROR REPLICAS RESOLUTION SIZE HW LEO LRS FOLLOWER OFFSETS\n mirror-topic 0 5001 docker-remote [] Online 0 B 0 0 0 0 []\n"})}),"\n",(0,o.jsx)(r.h3,{id:"generate-metadata-for-remote-clusters",children:"Generate Metadata for Remote Clusters"}),"\n",(0,o.jsx)(r.p,{children:"Each remote cluster requires a unique metadata file that gives the remote cluster the information to connect to the home cluster and the topic/mirror where the data is synchronized."}),"\n",(0,o.jsx)(r.p,{children:"Generate a metadata file for the remote:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio remote export docker-remote --file /shared/docker-remote.json --public-endpoint sc-home:9003\n"})}),"\n",(0,o.jsx)(r.h1,{id:"connect-to-the-home-cluster-from-the-remote",children:"Connect to the Home Cluster from the Remote"}),"\n",(0,o.jsx)(r.p,{children:"Now, connect to the remote shell in another terminal:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"docker exec -it sc-remote /bin/sh\n"})}),"\n",(0,o.jsx)(r.p,{children:"Then, we'll use the metadata file to connect to home:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio home connect --file /shared/docker-remote.json\n"})}),"\n",(0,o.jsx)(r.p,{children:"Let's check the partitions:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio partition list\n"})}),"\n",(0,o.jsx)(r.p,{children:"The remote device should show the following partition::"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:" TOPIC PARTITION LEADER MIRROR REPLICAS RESOLUTION SIZE HW LEO LRS FOLLOWER OFFSETS\n mirror-topic 0 5001 home:5001:spu-home:9010 [] Online 0 B 0 0 0 0 []\n"})}),"\n",(0,o.jsx)(r.p,{children:"Also, check the home status with:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio home status\n"})}),"\n",(0,o.jsx)(r.p,{children:"It should show the following:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:" REMOTE SC STATUS SPU
1STATUS LAST SEEN ERRORS\n home Connected Connected 1s -\n"})}),"\n",(0,o.jsx)(r.hr,{}),"\n",(0,o.jsx)(r.h2,{id:"producing-and-consuming-on-mirroring",children:"Producing and Consuming on Mirroring"}),"\n",(0,o.jsx)(r.p,{children:"Let's produce on the remote and consume from the home cluster."}),"\n",(0,o.jsx)(r.h3,{id:"produce-to-remote-cluster",children:"Produce to remote cluster"}),"\n",(0,o.jsx)(r.p,{children:"On remote shell, produce with:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio produce mirror-topic\n"})}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"> A\nOk!\n> B\nOk!\n"})}),"\n",(0,o.jsx)(r.h3,{id:"consume-from-home-cluster",children:"Consume from Home cluster"}),"\n",(0,o.jsx)(r.p,{children:"On home shell, consume with:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"fluvio consume mirror-topic --mirror docker-remote -B\n"})}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-bash",children:"A\nB\n"})}),"\n",(0,o.jsx)(r.p,{children:"\ud83c\udf89 Congratulations! You have successfully tested mirroring."}),"\n",(0,o.jsx)(r.h4,{id:"related",children:"Related"}),"\n",(0,o.jsxs)(r.ul,{children:["\n",(0,o.jsx)(r.li,{children:(0,o.jsx)(r.a,{href:"/docs/0.15.0/cloud/tutorials/try-mirroring-cloud",children:"Try Mirroring - Cloud"})}),"\n",(0,o.jsx)(r.li,{children:(0,o.jsx)(r.a,{href:"/docs/0.15.0/cloud/tutorials/iot-mirroring-cloud",children:"IoT Mirroring - Cloud"})}),"\n",(0,o.jsx)(r.li,{children:(0,o.jsx)(r.a,{href:"/docs/0.15.0/fluvio/tutorials/mirroring-iot-local",children:"IoT Mirroring - Raspberry Pi to a remote"})}),"\n",(0,o.jsx)(r.li,{children:(0,o.jsx)(r.a,{href:"https://discord.com/invite/bBG2dTz",children:"Discord"})}),"\n"]})]})}function h(e={}){const{wrapper:r}={...(0,t.R)(),...e.components};return r?(0,o.jsx)(r,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},28453:(e,r,n)=>{n.d(r,{R:()=>s,x:()=>l});var o=n(96540);const t={},i=o.createContext(t);function s(e){const r=o.useContext(i);return o.useMemo((function(){return"function"==typeof e?e(r):{...r,...e}}),[r,e])}function l(e){let r;return r=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:s(e.components),o.createElement(i.Provider,{value:r},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.