1"use strict";(self.webpackChunkcilium_io=self.webpackChunkcilium_io||[]).push([[5803],{6452:function(e,a){a.A={thomasGraf:{header:"Thomas Graf",bio:'Thomas Graf is a Co-Founder of Cilium and the CTO & Co-Founder of <a href="https://isovalent.com/?utm_source=website-cilium&utm_medium=referral&utm_campaign=cilium-enterprise">Isovalent</a>, the company behind Cilium. Before that, Thomas spent 15 years as\n a kernel developer working on the <a href="https://kernel.org">Linux kernel</a> in networking, security and eventually eBPF.'},lizRice:{header:'<a href="https://twitter.com/lizrice">Liz Rice</a>',bio:'Liz is Chief Open Source Officer at <a href="https://isovalent.com/?utm_source=website-cilium&utm_medium=referral&utm_campaign=cilium-enterprise" target="_blank" rel="noopener noreferrer">Isovalent</a>, the company behind Cilium. She is also chair of the CNCF\'s Technical Oversight Committee, and the author of Container Security published by O\'Reilly.'},luanGuimaraes:{header:"Luan Guimarães",bio:"Luan is a Brazilian rock climber, amateur musician, and\n programmer and am enthusiastic about free software communities and other\n open knowledge initiatives. He has been working as a Site Reliability\n Engineer at Wildlife Studios, using and building infrastructure tools on\n top of Kubernetes in order to support millions of users around the world."},joshVanLeeuwen:{header:"Josh Van Leeuwen",bio:"Josh interned at Jetstack during the summer of 2017 before continuing to\n work part time during his final year of study at the University of Bristol.\n During this year, Josh developed a Kubernetes custom controller that\n automates the delegation of RBAC permissions based on time and event\n triggers. This work was later awarded the best Software Development Tool\n Final Year Project. Josh now works full time at Jetstack where if heâs not\n writing more Go, heâs making good food."},howardHao:{header:"Howard Hao",bio:" Howard Hao has been working as a Site Reliability Engineer for five years at\n Ect888.com since graduating from Shanghai Jiao Tong University. His team\n consists of 7 members and has been focusing on the construction of\n container orchestration platform like Kubernetes for one and a half years."},sergeyGeneralov:{header:"Sergey Generalov",bio:"Sergey is a member of the technical staff at Isovalent\n and focuses on helping Cilium users solve challenges related\n to network policies, monitoring, and connectivity troubleshooting\n by building tools like Network Policy Editor, Hubble UI and more."},liWenquan:{header:"Li Wenquan",bio:"Hello everyone, I am Li Wenquan from China. You can call me David. I\n started my Docker journey from 2014 and now work as a project manager of\n enterprise container platform, which is built on Kubernetes and Mesos. I\n got to know Cilium project from Kubecon, it is so interesting and\n promising. I've learned a lot from it, such as BPF, XDP and how to replace\n kube-proxy in a elegant way and I'd love to contribute to it."},alexanderAlemayhu:{header:"Alexander Alemayhu",bio:"Alexander Alemayhu is a software engineer at Isovalent,\n the company behind Cilium. He has been working on eBPF and Linux\n kernel technologies for several years, focusing on networking and observability solutions."},DanielBorkmann:{header:"Daniel Borkmann",bio:"Daniel Borkmann is a Distinguished Software Engineer, Isovalent at Cisco"},ThomasGraf:{header:"Thomas Graf",bio:"Thomas Graf is the CTO & Co-Founder Isovalent and also the Vice President Security Cisco"},JedSalazar:{header:"Jed Salazar",bio:"Jed Salazar is a Senior Solutions Architect, Isovalent"},JedSalazarandJoeStringer:{header:"Jed Salazar and Joe Stringer",bio:"Jed Salazar is a Senior Solutions Architect at Isovalent\n and Joe Stringer is a Principal Engineer, Isovalent at Cisco"},JosephIrving:{header:"Joseph Irving",bio:"Joseph Irving is a Platform Engineer Lead at RVU (Uswitch)"},BillMulligan:{header:"Bill Mulligan",bio:"Bill Mulligan is a Cilium and eBPF Community Pollinator,\n Isovalent at Cisco and a Governing Board Member of the eBPF Foundation."},OndrejBlazek:{header:"Ondrej Blazek",bio:"Ondrej Blazek is an Infrastru
1cture Engineer at Seznam.cz"},LeonardCohnenandMoritzEckert:{header:"Leonard Cohnen and Moritz Eckert",bio:"Leonard Cohnen and Moritz Eckert are team members at Edgeless Systems"},PolArroyo:{header:"Pol Arroyo",bio:"Pol Arroyo is a DevOps Engineer at Hetzner Cloud."},JedSalazarandMartynasPumputis:{header:"Jed Salazar and Martynas Pumputis",bio:"Jed Salazar is a Senior Solutions Architect, Isovalent and Martynas Pumputis is a Principal Software Engineer, Isovalent at Cisco"},ShedrackAkintayo:{header:"Shedrack Akintayo",bio:"Shedrack Akintayo is a Community Manager at\n Isovalent helping build the eBPF and Cilium open source communities"},AmirKheirkhahan:{header:"Amir Kheirkhahan",bio:"Amir Kheirkhahan is a DevOps Specialist at DB Schenker handling design, development,\n deployment and maintenance of wide range of devops toolchain on top of Kubernetes clusters"},PaulArah:{header:"Paul Arah",bio:"Paul Arah is a Community Builder focused on Security at Isovalent (Cisco)"},HimalKumar:{header:"Himal Kumar, Bhaskar Dutta, Arman Pashamokhtari",bio:"Himal Kumar, Bhaskar Dutta, Arman Pashamokhtari are all part of the\n CanopusAI team Real Time Network Observability, powered by eBPF and Agentic AI"},DoniaChaiehloudj:{header:"Donia Chaiehloudj",bio:"Donia Chaiehloudj is a Senior Software Engineer and Community Oriented at Isovalent.\n She has been working on Cilium and eBPF technologies, focusing on networking and security solutions."},KatieMeinders:{header:"Katie Meinders",bio:"Katie Meinders is a Community Builder at Isovalent where\n she helps grow the Cilium and eBPF communities through storytelling,\n social media, showcasing user success, and building connections across the open source ecosystem."},PeaceSandy:{header:"Peace Sandy",bio:"Peace Sandy is an LFX mentee who contributed to improving Cilium SEO, AEO, and\n AIO during her mentorship period."},NehaAggarwal:{header:"Neha Aggarwal",bio:"Neha Aggarwal is a Principal Engineer at Microsoft."},CharityMbisi:{header:"Charity Mbisi",bio:"Charity Mbisi is an LFX mentee who contributed to improving Cilium's SEO, AEO, and AIO during his mentorship period.\n Professionally, Charity Mbisi is a Software Engineer consulting in the Fin-tech and banking industry, specializing in building cloud native computing solutions and optimized service delivery."},andreMartinsAndFerozSalam:{header:"André Martins and Feroz Salam",bio:"André Martins is a Cilium maintainer and Software Engineer, Isovalent at Cisco.\n Feroz Salam is a member of the Cilium Security Team and a Security Engineer, Isovalent at Cisco."},ChristianHernandez:{header:"Christian Hernandez",bio:"Christian is a well rounded technologist with experience in infrastructure engineering, systems administration, enterprise architecture, tech support, advocacy, and product management. Passionate about OpenSource and containerizing the world one application at a time. He is currently a maintainer of the Argo Project and OpenGitops. Currently, he works as a Technical Marketing Engineer and Tech Lead at Cisco. He focuses on GitOps practices, DevOps, Kubernetes, Network Security, and Containers."},AkilaInduranga:{header:"Akila Induranga",bio:'Akila is a Senior Software Engineer at WSO2, and a maintainer of <a href="https://openchoreo.dev/" target="_blank" rel="noopener noreferrer">OpenChoreo</a>, an open-source internal developer platform for Kubernetes and a CNCF sandbox project.\n He works on the platform\'s observability and networking layers, including the Cilium-based networking module that brings identity-based policy and Hubble observability to OpenChoreo cells.'}}},7455:function(e,a,n){n.r(a),n.d(a,{Head:function(){return m},default:function(){return g}});var t=n(8453),i=n(6540),s=n(6452);function o(e){const a=Object.assign({p:"p",em:"em",span:"span",h2:"h2",a:"a",h3:"h3",ul:"ul",li:"li",strong:"strong"},(0,t.RP)(),e.components),{BlogAuthor:n}=a;return n||function(e,a){throw new Error("Expected "+(a?"component":"object")+" `"+e+"` to be defined: you likely forgot to import, pass, or provide it.")}("BlogAuthor",!0),i.createElement(i.Fragment,null,i.createElement(a.p,null,i.createElement(a.em,null,"Author: Amir Kheirkhahan, DB Schenker")),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/e43de7ad0a1ebea054aa540062c4e0e0/0d4f8/after-removing-calico-resources.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 30.555555555555554%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/e43de7ad0a1ebea054aa540062c4e0e0/2ff5b/after-removing-calico-resources.webp 252w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/4d583/after-removing-calico-resources.webp 504w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/905a7/after-removing-calico-resources.webp 1008w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/bb9f8/after-removing-calico-resources.webp 1512w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/485a2/after-removing-calico-resources.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/e43de7ad0a1ebea054aa540062c4e0e0/019e0
1/after-removing-calico-resources.png 252w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/0dcb2/after-removing-calico-resources.png 504w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/832a9/after-removing-calico-resources.png 1008w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/19357/after-removing-calico-resources.png 1512w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/0d4f8/after-removing-calico-resources.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/e43de7ad0a1ebea054aa540062c4e0e0/832a9/after-removing-calico-resources.png"\n alt="After removing calico resources"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("a",{id:"history-behind-the-migration-to-cilium"}),"\n",i.createElement(a.h2,null,"History behind the Migration to Cilium"),"\n",i.createElement(a.p,null,"In the past, the IT unit for the land transportation business of ",i.createElement(a.a,{href:"https://www.dbschenker.com/de-de"},"DB Schenker")," used Calico as a Container Network Interface (",i.createElement(a.a,{href:"https://www.cni.dev/"},"CNI"),") for in-Kubernetes-Cluster communication, like pod-to-pod communication. Recently our team had the chance to participate in the 2023 ",i.createElement(a.a,{href:"https://events.linuxfoundation.org/kubecon-cloudnativecon-europe/"},"KubeCon")," in Amsterdam, where we learned a lot about ",i.createElement(a.a,{href:"https://ebpf.io/"},"eBPF")," and especially ",i.createElement(a.a,{href:"https://cilium.io/"},"Cilium"),", which was an important driver for us to question our CNI strategy."),"\n",i.createElement(a.p,null,"Although Calico offers eBPF as well and leverages some important features of it, Cilium was built natively on top of eBPF. In parallel we saw wide adoption in the market and a very feature rich tooling and ecosystem around Cilium, like ",i.createElement(a.a,{href:"https://github.com/cilium/tetragon"},"Tetragon")," for security observability and ",i.createElement(a.a,{href:"https://github.com/cilium/hubble"},"Hubble")," for network visibility. All of these factors together made the decision clear that we needed to migrate to Cilium to prepare our platform for the next steps in our cloud native journey."),"\n",i.createElement("a",{id:"live-migration-considerations"}),"\n",i.createElement(a.h2,null,"Live migration Considerations"),"\n",i.createElement(a.p,null,"When planning the migration of all our Kubernetes clusters, we needed to architect it to have minimal down time."),"\n",i.createElement(a.p,null,"To understand how we could do this, we started a POC and tried to use the capabilities of ",i.createElement(a.a,{href:"https://github.com/k8snetworkplumbingwg/multus-cni"},"Multus")," to have multiple network interfaces attached in parallel to a Pod. Unfortunately, this option didnât work for us due to the complexity and risk of longer downtime."),"\n",i.createElement(a.p,null,"However, Cilium recently introduced support for a ",i.createElement(a.a,{href:"https://docs.cilium.io/en/stable/installation/k8s-install-migration/#migration-via-dual-overlays"},"hybrid mode")," for migration where 2 different CNIs with different CIDR-ranges can be established across the cluster, so we found a good new motivation to give it a chance."),"\n",i.createElement(a.p,null,"As already stated for the Land Business, it was important to keep the downtime of clusters minimal. Cilium has a brilliant feature which helped us to achieve this goal. It offers a ",i.createElement(a.a,{href:"https://docs.cilium.io/en/latest/configuration/per-node-config/"},"per-node configuration")," feature to easily roll out the Cilium configuration on a node by node basis using node-labels. That means, after you install Cilium and apply the configuration, all new nodes identified with a matching label will automatically get the specified configuration. This ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">CiliumNodeConfig</code>'}})," object has been available since ",i.createElement(a.a,{href:"https://isovalent.com/blog/post/cilium-release-113/#resilience-and-troubleshooting"},"Cilium 1.13"),". We will go through this feature later during the migration steps."),"\n",i.createElement(a.p,null,"The other aspect we needed to consider was how to replace the current Calico IP-in-IP tunneling protocol. Cilium offers UDP-based ",i.createElement(a.a,{href:"https://docs.cilium.io/en/stable/network/concepts/routing/"},"encapsulation protocols")," VXLAN or GENEVE and based on our needs we chose VXLAN where the entire layer 2 Ethernet frame is encapsulated inside a UDP packet and transmitted over the node network."),"\n",i.createElement(a.p,null,"For IP address distribution among pods, you have different ",i.createElement(a.a,{href:"https://docs.cilium.io/en/stable/network/concepts/ipam"},"options")," to select in Cilium. Cluster-scope IPAM mode is the recommended way which we also chose. In this case, the Cilium operator manages the IP addresses by creating a CiliumNode object for each node and assigning the podCIDRs to them which is then read by Cilium agents running on the nodes. Of course, you are not allowed to select the same podCIDR used by Calico."),"\n",i.createElement(a.p,null,"Another important consideration was the kube-proxy replacement feature. When a new service is created, the API server notifies all kube-proxy agents running on the worker nodes about this event. Then kube-proxy creates iptable rules to make sure each packet destined for a service is forwarded to the one of the correct backend pods. It is the same for a new pod when the endpoints in iptables are updated. In the following migration steps, we kept Ciliumâs kube-proxy replacement option disabled to reduce the complexity of the migration."),"\n",i.createElement(a.p,null,"Finally, we also needed to think about our Kafka clusters. We run Kafka inside our Kubernetes environment by using the ",i.createElement(a.a,{href:"https://strimzi.io/"},"strimzi")," chart which includes very sensitive and important data. To ensure the stability of Kafka, we needed to prevent losing the cluster state or running into a Kafka leader election issue."),"\n",i.createElement(a.p,null,"To get a better understanding of our infrastru
1cture, Iâll shortly point out some tooling and approaches which we use within DB Schenker. We use self-managed Kubernetes clusters on AWS where the worker and controller nodes are running independently on separate VMs. The worker nodes run on a mix of spot and on-demand instances. The base Images are built by Packer and Ansible and the whole infrastructure setup is managed by Terraform. To meet IT Security requirements, every day we throw away a set of our worker and controller nodes. We also have automated processes to rotate all our nodes. We do this by starting a new node and waiting until it is ready, then we drain the old node and move all workloads away from it before terminating the old node in our cloud provider as well as in Kubernetes. These steps are repeated for each of our worker pools."),"\n",i.createElement(a.p,null,"For monitoring and measuring connectivity between nodes we leverage ",i.createElement(a.a,{href:"https://github.com/bloomberg/goldpinger"},"Goldpinger")," and for Kafka monitoring we leverage ",i.createElement(a.a,{href:"https://github.com/strimzi/strimzi-canary"},"strimzi canary"),", which is a component of the ",i.createElement(a.a,{href:"https://github.com/strimzi/strimzi-kafka-operator"},"strimzi kafka operator.")),"\n",i.createElement(a.p,null,"The overview of the migration is as follows and we will go through these steps in the next section:"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/d94f6ed5df7326e6f6fd1795cfc41086/0d4f8/existing-nodes-with-calico.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 46.42857142857143%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/d94f6ed5df7326e6f6fd1795cfc41086/2ff5b/existing-nodes-with-calico.webp 252w,\n/static/d94f6ed5df7326e6f6fd1795cfc41086/4d583/existing-nodes-with-calico.webp 504w,\n/static/d94f6ed5df7326e6f6fd1795cfc41086/905a7/existing-nodes-with-calico.webp 1008w,\n/static/d94f6ed5df7326e6f6fd1795cfc41086/bb9f8/existing-nodes-with-calico.webp 1512w,\n/static/d94f6ed5df7326e6f6fd1795cfc41086/485a2/existing-nodes-with-calico.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/d94f6ed5df7326e6f6fd1795cfc41086/019e0/existing-nodes-with-calico.png 252w,\n/static/d94f6ed5df7326e6f6fd1795cfc41086/0dcb2/existing-nodes-with-calico.png 504w,\n/static/d94f6ed5df7326e6f6fd1795cfc41086/832a9/existing-nodes-with-calico.png 1008w,\n/static/d94f6ed5df7326e6f6fd1795cfc41086/19357/existing-nodes-with-calico.png 1512w,\n/static/d94f6ed5df7326e6f6fd1795cfc41086/0d4f8/existing-nodes-with-calico.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/d94f6ed5df7326e6f6fd1795cfc41086/832a9/existing-nodes-with-calico.png"\n alt="existing nodes with Calico"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("div",{align:"center",style:{fontStyle:"italic"}},"Figure 1. Existing Nodes with Calico CNI"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/0d4f8/after-cilium-installation.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 46.42857142857143%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/2ff5b/after-cilium-installation.webp 252w,\n/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/4d583/after-cilium-installation.webp 504w,\n/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/905a7/after-cilium-installation.webp 1008w,\n/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/bb9f8/after-cilium-installation.webp 1512w,\n/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/485a2/after-cilium-installation.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/019e0/after-cilium-installation.png 252w,\n/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/0dcb2/after-cilium-installation.png 504w,\n/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/832a9/after-cilium-installation.png 1008w,\n/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/19357/after-cilium-installation.png 1512w,\n/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/0d4f8/after-cilium-installation.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/4f9d1e5c66e0b3bff223b174d8c5a8e3/832a9/after-cilium-installation.png"\n alt="After Cilium installation"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("div",{align:"center",style:{fontStyle:"italic"}},"Figure 2. After Cilium Installation on exisiting Nodes, Cilium has still no CNI ownership"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/9e50d09d29b13c991f810b1b51b1abdb/0d4f8/after-starting-the-rotation.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 30.555555555555554%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/9e50d09d29b13c991f810b1b51b1abdb/2ff5b/after-starting-the-rotation.webp 252w,\n/static/9e50d09d29b13c991f810b1b51b1abdb/4d583/after-starting-the-rotation.webp 504w,\n/static/9e50d09d29b13c991f810b1b51b1abdb/905a7/after-starting-the-rotation.webp 1008w,\n/static/9e50d09d29b13c991f810b1b51b1abdb/bb9f8/after-starting-the-rotation.webp 1512w,\n/static/9e50d09d29b13c991f810b1b51b1abdb/485a2/after-starting-the-rotation.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/9e50d09d29b13c991f810b1b51b1abdb/019e0/after-starting-the-rotation.png 252w,\n/static/9e50d09d29b13c991f810b1b51b1abdb/0dcb2/after-starting-the-rotation.png 504w,\n/static/9e50d09d29b13c991f810b1b51b1abdb/832a9/after-starting-the-rotation.png 1008w,\n/static/9e50d09d29b13c991f810b1b51b1abdb/19357/after-starting-the-rotation.png 1512w,\n/static/9e50d09d29b13c991f810b1b51b1abdb/0d4f8/after-starting-the-rotation.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/9e50d09d29b13c991f810b1b51b1abdb/832a9/after-starting-the-rotation.png"\n alt="After Starting the Rotation"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("div",{align:"center",style:{fontStyle:"italic"}},"Figure 3. After Starting the rotation, Node C starts with both CNIs and Cilium has CNI ownership. Node C currently has Cilium configurations in place which is discussed later at step 1 of migration"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/d996557a6599dff5678f1c06a1e3d471/0d4f8/draining-the-oldest-worker-node.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 30.555555555555554%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/d996557a6599dff5678f1c06a1e3d471/2ff5b/draining-the-oldest-worker-node.webp 252w,\n/static/d996557a6599dff5678f1c06a1e3d471/4d583/draining-the-oldest-worker-node.webp 504w,\n/static/d996557a6599dff5678f1c06a1e3d471/905a7/draining-the-oldest-worker-node.webp 1008w,\n/static/d996557a6599dff5678f1c06a1e3d471/bb9f8/draining-the-oldest-worker-node.webp 1512w,\n/static/d996557a6599dff5678f1c06a1e3d471/485a2/draining-the-oldest-worker-node.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/d996557a6599dff5678f1c06a1e3d471/019e0/draining-the-oldest-worker-node.png 252w,\n/static/d996557a6599dff5678f1c06a1e3d471/0dcb2/draining-the-oldest-worker-node.png 504w,\n/static/d996557a6599dff5678f1c06a1e3d471/832a9/draining-the-oldest-worker-node.png 1008w,\n/static/d996557a6599dff5678f1c06a1e3d471/19357/draining-the-oldest-worker-node.png 1512w,\n/static/d996557a6599dff5678f1c06a1e3d471/0d4f8/draining-the-oldest-worker-node.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/d996557a6599dff5678f1c06a1e3d471/832a9/draining-the-oldest-worker-node.png"\n alt="Draining the oldest worker node"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("div",{align:"center",style:{fontStyle:"italic"}},"Figure 4. Draining the older worker node (here Node A) and re-schdule the pods on the new Node"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/dadf9ae2113dee6ecfce651f7170698e/0d4f8/node-a-fully-replaced.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 30.952380952380953%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/dadf9ae2113dee6ecfce651f7170698e/2ff5b/node-a-fully-replaced.webp 252w,\n/static/dadf9ae2113dee6ecfce651f7170698e/4d583/node-a-fully-replaced.webp 504w,\n/static/dadf9ae2113dee6ecfce651f7170698e/905a7/node-a-fully-replaced.webp 1008w,\n/static/dadf9ae2113dee6ecfce651f7170698e/bb9f8/node-a-fully-replaced.webp 1512w,\n/static/dadf9ae2113dee6ecfce651f7170698e/485a2/node-a-fully-replaced.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/dadf9ae2113dee6ecfce651f7170698e/019e0/node-a-fully-replaced.png 252w,\n/static/dadf9ae2113dee6ecfce651f7170698e/0dcb2/node-a-fully-replaced.png 504w,\n/static/dadf9ae2113dee6ecfce651f7170698e/832a9/node-a-fully-replaced.png 1008w,\n/static/dadf9ae2113dee6ecfce651f7170698e/19357/node-a-fully-replaced.png 1512w,\n/static/dadf9ae2113dee6ecfce651f7170698e/0d4f8/node-a-fully-replaced.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/dadf9ae2113dee6ecfce651f7170698e/832a9/node-a-fully-replaced.png"\n alt="Node A fully replaced"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("div",{align:"center",style:{fontStyle:"italic"}},"Figure 5. Node A is removed from the LoadBalancer so that it no longer receives traffic, and then terminated. Node A is fully replaced by Node C"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/0d4f8/after-the-full-rotation.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 46.42857142857143%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/2ff5b/after-the-full-rotation.webp 252w,\n/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/4d583/after-the-full-rotation.webp 504w,\n/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/905a7/after-the-full-rotation.webp 1008w,\n/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/bb9f8/after-the-full-rotation.webp 1512w,\n/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/485a2/after-the-full-rotation.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/019e0/after-the-full-rotation.png 252w,\n/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/0dcb2/after-the-full-rotation.png 504w,\n/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/832a9/after-the-full-rotation.png 1008w,\n/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/19357/after-the-full-rotation.png 1512w,\n/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/0d4f8/after-the-full-rotation.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/78ae04ad8b38a3c5c6fbc3bbb6ba2eb4/832a9/after-the-full-rotation.png"\n alt="After the Full rotation"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("div",{align:"center",style:{fontStyle:"italic"}},"Figure 6. After the full rotation of both Nodes, both CNIs should be running on new nodes with Cilium leading"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/e43de7ad0a1ebea054aa540062c4e0e0/0d4f8/after-removing-calico-resources.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 30.555555555555554%; position: relative;
1bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/e43de7ad0a1ebea054aa540062c4e0e0/2ff5b/after-removing-calico-resources.webp 252w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/4d583/after-removing-calico-resources.webp 504w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/905a7/after-removing-calico-resources.webp 1008w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/bb9f8/after-removing-calico-resources.webp 1512w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/485a2/after-removing-calico-resources.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/e43de7ad0a1ebea054aa540062c4e0e0/019e0/after-removing-calico-resources.png 252w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/0dcb2/after-removing-calico-resources.png 504w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/832a9/after-removing-calico-resources.png 1008w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/19357/after-removing-calico-resources.png 1512w,\n/static/e43de7ad0a1ebea054aa540062c4e0e0/0d4f8/after-removing-calico-resources.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/e43de7ad0a1ebea054aa540062c4e0e0/832a9/after-removing-calico-resources.png"\n alt="After removing calico resources"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("div",{align:"center",style:{fontStyle:"italic"}},"Figure. 7. After removing Calico resources and clean up the labels, the new nodes have only Cilium installed on it"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/93a6df0c02caf1fcdba30b9276759755/0d4f8/only-cilium.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 47.22222222222222%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/93a6df0c02caf1fcdba30b9276759755/2ff5b/only-cilium.webp 252w,\n/static/93a6df0c02caf1fcdba30b9276759755/4d583/only-cilium.webp 504w,\n/static/93a6df0c02caf1fcdba30b9276759755/905a7/only-cilium.webp 1008w,\n/static/93a6df0c02caf1fcdba30b9276759755/bb9f8/only-cilium.webp 1512w,\n/static/93a6df0c02caf1fcdba30b9276759755/485a2/only-cilium.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/93a6df0c02caf1fcdba30b9276759755/019e0/only-cilium.png 252w,\n/static/93a6df0c02caf1fcdba30b9276759755/0dcb2/only-cilium.png 504w,\n/static/93a6df0c02caf1fcdba30b9276759755/832a9/only-cilium.png 1008w,\n/static/93a6df0c02caf1fcdba30b9276759755/19357/only-cilium.png 1512w,\n/static/93a6df0c02caf1fcdba30b9276759755/0d4f8/only-cilium.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/93a6df0c02caf1fcdba30b9276759755/832a9/only-cilium.png"\n alt="only-cilium"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("div",{align:"center",style:{fontStyle:"italic"}},"Figure 8. At the end, only Cilium is running on all nodes."),"\n",i.createElement(a.h2,null,"Live migration"),"\n",i.createElement(a.h3,null,"Step 1- Preparation"),"\n",i.createElement(a.p,null,"Before starting the migration, we need to make sure that we have proper back-ups of the workloads. In the Kubernetes context, the most important piece is to keep etcd backedup in a safe place, like an AWS S3 Bucket. In case the migration fails, we have a backup in place to perform a disaster recovery to the state before the migration."),"\n",i.createElement(a.p,null,"The next thing that should be prepared is building the migration base image. Some of the important changes are:"),"\n",i.createElement(a.ul,null,"\n",i.createElement(a.li,null,"\n",i.createElement(a.p,null,"Add the label ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">"io.cilium.migration/cilium-default=true"</code>'}})," to all newly created nodes. As soon as a new node is launched and has this label, the predefined Cilium config will be applied to this node. These predefined configs are declared in the ",i.createElement(a.a,{href:"https://docs.cilium.io/en/latest/configuration/per-node
1-config/#ciliumnodeconfig-objects"},"CiliumNodeConfig")," object."),"\n"),"\n",i.createElement(a.li,null,"\n",i.createElement(a.p,null,"Add the Taint ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">"node.cilium.io/agent-not-ready=:NoSchedule.</code>'}})," to all newly created nodes. This prevents a race condition between the CNIs that are now running in parallel on the new nodes. Since Calico is sometimes faster in starting, the node gets marked as ready and newly scheduled pods on the node get IP addresses assigned by Calico. With the taint, Cilium is able to start up properly and then take over assignment of IPs.\nFor more information refer to ",i.createElement(a.a,{href:"https://docs.cilium.io/en/latest/installation/taints"},"official Cilium documentation.")),"\n"),"\n"),"\n",i.createElement(a.p,null,"You should make your changes ready for deployment after Cilium installation. In our case, we built a new AMI with these changes and kept them ready to deploy. We used Terraform for deploying the infrastructure in our cloud environment and in this context, we rolled out the new images right after the Cilium installation. With that all new nodes have this label and taint and Cilium is the leading CNI on those nodes."),"\n",i.createElement(a.h3,null,"Step 2- Scale down the important applications like Strimzi Kafka"),"\n",i.createElement(a.p,null,"During a migration, if you arenât careful with the stateful components, like Kafka, you could lose data. Therefore, we scale down Kafka and Zookeeper to ensure no messages are produced or consumed."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="bash"><pre class="language-bash"><code class="language-bash"> kubectl scale statefulset <span class="token parameter variable">-n</span> kafka kafka-broker <span class="token parameter variable">--replicas</span> <span class="token number">0</span>\n kubectl scale statefulset <span class="token parameter variable">-n</span> kafka kafka-zookeeper <span class="token parameter variable">--replicas</span> <span class="token number">0</span></code></pre></div>'}}),"\n",i.createElement(a.p,null,"This does cause a short downtime for our workloads but it helped us prevent issues with one of our most important components."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="bash"><pre class="language-bash"><code class="language-bash"> kubectl drain --ignore-daemonsets --delete-emptydir-data <span class="token parameter variable">--force</span> --grace-period<span class="token operator">=</span><span class="token number">900</span> <span class="token parameter variable">-l</span> <span class="token assign-left variable">label</span><span class="token operator">=</span>gp-kafka</code></pre></div>'}}),"\n",i.createElement(a.p,null,"As soon as Kafka is completely stopped, we terminate the underlying node pool to be able to quickly bring up the nodes with Cilium installed and configured properly in the cluster."),"\n",i.createElement(a.h3,null,"Step 3- Cilium installation"),"\n",i.createElement(a.p,null,"Afterwards, we apply the changes to Terraform that contain the labels and taints that are required to tell Cilium to run as leading CNI on those nodes. It is important to mention that this step contains no new node deployment and is rather a preparation for the final installation and configuration of Cilium. Node deployment happens during the rotation."),"\n",i.createElement(a.p,null,"Now it is the time to install Cilium where the configuration looks like this:"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="yaml"><pre class="language-yaml"><code class="language-yaml"><span class="token comment"># migration, for connectivity between calico and cilium</span>\n\n bpf<span class="token punctuation">:</span>\n\n hostLegacyRouting<span class="token punctuation">:</span> true \n\n<span class="token comment"># Set cluster name. It would be interesting for cluster mesh</span>\n\n cluster<span class="token punctuation">:</span>\n\n id<span class="token punctuation">:</span> <span class="token number">0</span>\n\n name<span class="token punctuation">:</span> prod<span class="token punctuation">-</span>cluster\n\n cni<span class="token punctuation">:</span>\n\n<span class="token comment"># migration, disable CNI changes</span>\n\n customConf<span class="token punctuation">:</span> true \n\n<span class="token comment"># should be always false so it doesn\'t remove cilium</span>\n\n uninstall<span class="token punctuation">:</span> false \n\n<span class="token comment"># Assign a new CIDR for cilium</span>\n\n ipam<span class="token punctuation">:</span>\n\n operator<span class="token punctuation">:</span>\n\n clusterPoolIPv4PodCIDRList<span class="token punctuation">:</span>
1\n\n <span class="token punctuation">-</span> 10.x.x.x/16 \n\n<span class="token comment"># migration, otherwise all calico pods are restarted and we get downtime</span>\n\n operator<span class="token punctuation">:</span>\n\n unmanagedPodWatcher<span class="token punctuation">:</span> \n\n restart<span class="token punctuation">:</span> <span class="token boolean important">false</span>\n\n<span class="token comment"># migration</span>\n\n policyEnforcementMode<span class="token punctuation">:</span> never \n\n<span class="token comment"># We stay with kubeproxy</span>\n\n kubeProxyReplacement<span class="token punctuation">:</span> disabled \n\n<span class="token comment"># Use Cilium suggested tunnel port 8473</span>\n\n tunnel<span class="token punctuation">:</span> vxlan\n\n tunnelPort<span class="token punctuation">:</span> <span class="token number">8473</span>\n\n<span class="token comment"># (Optional) to resolve the sonobuoy test</span>\n\n sessionAffinity<span class="token punctuation">:</span> true \n\n<span class="token comment"># (Optional) Enable prometheus</span>\n\n prometheus<span class="token punctuation">:</span>\n\n enabled<span class="token punctuation">:</span> <span class="token boolean important">true</span>\n\n metrics<span class="token punctuation">:</span> \\~\n\n operator<span class="token punctuation">:</span>\n\n prometheus<span class="token punctuation">:</span>\n\n enabled<span class="token punctuation">:</span> <span class="token boolean important">true</span>\n\n<span class="token comment"># (Optional) Enable Hubble</span>\n\n hubble<span class="token punctuation">:</span>\n\n relay<span class="token punctuation">:</span>\n\n enabled<span class="token punctuation">:</span> <span class="token boolean important">true</span>\n\n prometheus<span class="token punctuation">:</span>\n\n enabled<span class="token punctuation">:</span> <span class="token boolean important">true</span>\n\n ui<span class="token punctuation">:</span>\n\n enabled<span class="token punctuation">:</span> <span class="token boolean important">true</span>\n\n metrics<span class="token punctuation">:</span>\n\n enabled<span class="token punctuation">:</span>\n\n <span class="token punctuation">-</span> dns\n\n <span class="token punctuation">-</span> tcp\n\n <span class="token punctuation">-</span> httpV2\n\n endpointStatus<span class="token punctuation">:</span>\n\n enabled<span class="token punctuation">:</span> <span class="token boolean important">true</span>\n\n status<span class="token punctuation">:</span> <span class="token string">"policy"</span></code></pre></div>'}}),"\n",i.createElement(a.p,null,"You must consider the proper CIDR ranges for Cilium and Calico and the previously allocated CIDR range should not be used. Migration steps are marked with ",i.createElement(a.em,null,"#migration")," comments and they will be changed at the end."),"\n",i.createElement(a.p,null,"We can start Ciliumâs installation at this step using the Helm deployment or another preferred approach. We installed the Cilium using the official Cilium Helm chart. The Cilium operator and agents are started in parallel with the existing Calico CNI, which will cause a short downtime for your applications (see below to remediate this). The downtime for us was less than 2 minutes and it went very smoothly. Of course, it could differ in another setup and it should be tested during the POC. During Cilium installation if you look at the Goldpinger UI, it looks a bit terrifying:"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/06151198cd687ec1309ce502aedb2842/0d4f8/goldpinger-ui.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 90.47619047619047%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/06151198cd687ec1309ce502aedb2842/2ff5b/goldpinger-ui.webp 252w,\n/static/06151198cd687ec1309ce502aedb2842/4d583/goldpinger-ui.webp 504w,\n/static/06151198cd687ec1309ce502aedb2842/905a7/goldpinger-ui.webp 1008w,\n/static/06151198cd687ec1309ce502aedb2842/bb9f8/goldpinger-ui.webp 1512w,\n/static/06151198cd687ec1309ce502aedb2842/485a2/goldpinger-ui.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/06151198cd687ec1309ce502aedb2842/019e0/goldpinger-ui.png 252w,\n/static/06151198cd687ec1309ce502aedb2842/0dcb2/goldpinger-ui.png 504w,\n/static/06151198cd687ec1309ce502aedb2842/832a9/goldpinger-ui.png 1008w,\n/static/06151198cd687ec1309ce502aedb2842/19357/goldpinger-ui.png 1512w,\n/static/06151198cd687ec1309ce502aedb2842/0d4f8/goldpinger-ui.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/06151198cd687ec1309ce502aedb2842/832a9/goldpinger-ui.png"\n alt="Goldpinger UI"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement("div",{align:"center",style:{fontStyle:"italic"}},"Figure 9. Goldpinger UI after Cilium installation"),"\n",i.createElement(a.p,null,"It turns out that this outage can be avoided altogether: when you install Cilium on a node, it creates a new network interface called ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">cilium\\_host</code>'}}),". If Calico decides to use that interface as its default interface, Calico node routing will start failing. For this reason, Calico needs to be configured to ignore the ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">cilium\\_host</code>'}})," interface. This could be done with the ",i.createElement(a.a,{href:"https://docs.tigera.io/calico/latest/reference/configure-calico-node#skip-interfaceinterface-regex"},"skipInterface setting on the Tigera operator"),". See the Cilium docs ",i.createElement(a.a,{href:"https://github.com/cilium/cilium/pull/27666"},"update")," for more information."),"\n",i.createElement(a.h3,null,"Step 4- Deploy CiliumNodeConfig"),"\n",i.createElement(a.p,null,"After all inter-node communications are green and connectivity between the nodes has been recovered, you need to roll out the ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">CiliumNodeConfig</code>'}}),". This resource causes new nodes with the label ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">"io.cilium.migration/cilium-default=true"</code>'}}
1)," to receive Cilium configurations."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="bash"><pre class="language-bash"><code class="language-bash"> <span class="token function">cat</span> <span class="token operator"><<</span>EOF <span class="token operator">|</span> kubectl apply --server-side <span class="token parameter variable">-f</span> -\n apiVersion: cilium.io/v2alpha1\n kind: CiliumNodeConfig\n metadata:\n namespace: kube-system\n name: cilium-default\n spec:\n nodeSelector:\n matchLabels:\n io.cilium.migration/cilium-default: <span class="token string">"true"</span>\n defaults:\n write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist\n custom-cni-conf: <span class="token string">"false"</span>\n cni-chaining-mode: <span class="token string">"none"</span>\n cni-exclusive: <span class="token string">"true"</span>\n EOF</code></pre></div>'}}),"\n",i.createElement(a.p,null,i.createElement(a.strong,null,"HINT"),": It could happen that ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">_coredns_ pods</code>'}})," are restarted permanently after Cilium installation. The Cilium operator needs ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">_coredns_</code>'}})," for DNS resolution and it will try to restart the pods to be able to resolve the dns queries. Because we still have Calico as leader on the nodes on which ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">_coredns_</code>'}})," is running, the restarting goes to an infinite loop. To solve the issue, we added 3 new worker nodes for each AZ with new images built by us in step 1. After starting those nodes, Cilium takes over the CNI ownership and ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">_coredns_</code>'}})," pods are scheduled on those nodes. To avoid the ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">_cluster-autoscaler_</code>'}})," terminating these new nodes due to low resource usage, we tell the ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">_cluster-autoscaler_</code>'}}),"to not consider these nodes as candidates for termination. For that, we add this annotation to ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">_coredns_ pods</code>'}}),":"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="bash"><pre class="language-bash"><code class="language-bash"> kubectl annotate pod <span class="token parameter variable">-n</span> kube-system <span class="token parameter variable">-l</span> k8s-app<span class="token operator">=</span>kube-dns cluster-autoscaler.kubernetes.io/safe-to-evict<span class="token operator">=</span>false</code></pre></div>'}}),"\n",i.createElement(a.p,null,"Now if you check the status of Cilium, you will still see that no pods are managed by Cilium, except ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">_coredns_</code>'}})," pods. This is expected. Calico is still running on the node and has CNI ownership. The nodes should be drained, cordoned, and the new nodes created as described later."),"\n",i.createElement(a.h3,null,"Step 5- Scale up Kafka"),"\n",i.createElement(a.p,null,"Now you can scale up Kafkaâs underlying worker pools to the previous numbers in your cloud provider. New nodes should have both CNIs running, but Cilium will take over leadership. After that you can start Kafka applications again. First start Zookeeper and wait for it to be ready and then start the Kafka brokers."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="bash"><pre class="language-bash"><code class="language-bash"> kubectl scale statefulset <span class="token parameter variable">-n</span> kafka kafka-zookeeper <span class="token parameter variable">--replicas</span> <span class="token number">5</span>\n kubectl scale statefulset <span class="token parameter variable">-n</span> kafka kafka-broker <span class="token parameter variable">--replicas</span> <span class="token number">5</span></code></pre></div>'}}),"\n",i.createElement(a.h3,null,"Step 6- Rotate the nodes"),"\n",i.createElement(a.p,null,"Now it is the time to rotate all nodes with the new image which was built in step 1. The rotation will take some hours depen
1ding on the rotation mechanism as well as the number of nodes. At the end, each new node should have Cilium as a CNI in parallel to Calico and all pods must have a new IP address from the Cilium CIDR range."),"\n",i.createElement(a.p,null,"To check how many nodes are not yet rotated the following command can be executed:"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="bash"><pre class="language-bash"><code class="language-bash"> kubectl get nodes <span class="token parameter variable">-l</span> <span class="token string">\'!io.cilium.migration/cilium-default\'</span></code></pre></div>'}}),"\n",i.createElement(a.h3,null,"Step 7- Cleanup "),"\n",i.createElement(a.p,null,"We proceed with this step when the node rotation has been completed and Cilium is the main CNI on all nodes. You can check this with the ",i.createElement(a.em,null,"âcilium statusâ")," command or filter the pods based on the IP range."),"\n",i.createElement(a.p,null,"To check which pods have still Calico IPs (when assuming Calicoâs IP range is 192.168.0.0/16) run:"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="bash"><pre class="language-bash"><code class="language-bash"> kubectl get po <span class="token parameter variable">-o</span> wide <span class="token parameter variable">-A</span> <span class="token operator">|</span> <span class="token function">grep</span> <span class="token parameter variable">-e</span> <span class="token string">"192\\.168\\."</span></code></pre></div>'}}),"\n",i.createElement(a.p,null,"These pods got the Calico network after node rotation and are unmanaged by Cilium. They will be restarted once we perform the post migration step. . For us it happened for some daemonsets like goldpinger which were started before the Cilium agents could not be managed. If they are not critical pods, they can be restarted simply using the following command at your own responsibility:"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="bash"><pre class="language-bash"><code class="language-bash"> kubectl get po <span class="token parameter variable">-o</span> custom-columns<span class="token operator">=</span>NAMESPACE:.metadata.namespace,NAME:.metadata.name <span class="token parameter variable">-A</span> --no-headers<span class="token operator">=</span>true <span class="token parameter variable">-o</span> wide <span class="token operator">|</span> <span class="token function">grep</span> <span class="token parameter variable">-e</span> <span class="token string">"192\\.168\\."</span> <span class="token operator">|</span> <span class="token function">awk</span> <span class="token string">\'{print "-n "$1" "$2}\'</span> <span class="token operator">|</span> <span class="token function">xargs</span> <span class="token parameter variable">-L</span> <span class="token number">1</span> <span class="token parameter variable">-r</span> kubectl delete pod</code></pre></div>'}}),"\n",i.createElement(a.p,null,"If you check the Cilium status, you will see all nodes are managed by the Cilium CNI. Now we can remove all Calico CRDs and other leftovers."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="bash"><pre class="language-bash"><code class="language-bash"> kubectl delete crd bgpconfigurations.crd.projectcalico.org bgppeers.crd.projectcalico.org blockaffinities.crd.projectcalico.org caliconodestatuses.crd.projectcalico.org clusterinformations.crd.projectcalico.org felixconfigurations.crd.projectcalico.org globalnetworkpolicies.crd.projectcalico.org globalnetworksets.crd.projectcalico.org hostendpoints.crd.projectcalico.org ipamblocks.crd.projectcalico.org ipamconfigs.crd.projectcalico.org ipamhandles.crd.projectcalico.org ippools.crd.projectcalico.org ipreservations.crd.projectcalico.org kubecontrollersconfigurations.crd.projectcalico.org networkpolicies.crd.projectcalico.org networksets.crd.projectcalico.org\n kubectl delete <span class="token parameter variable">-n</span> kube-system deploy calico-kube-controllers\n kubectl delete <span class="token parameter variable">-n</span> kube-system ds/calico-node\n kubectl delete <span class="token parameter variable">-n</span> kube-system cm/calico-config\n kubectl delete <span class="token parameter variable">-n</span> kube-system ciliumnodeconfig cilium-default</code></pre></div>'}}),"\n",i.createElement(a.h3,null,"Step 8- Post migration steps "),"\n",i.createElement(a.p,null,"As the ",i.createElement(a.a,{href:"https://docs.cilium.io/en/stable/installation/k8s-install-migration/#post-migration"},"official documentation")," suggested, we do some post migration steps, like restarting the unmanaged pods by cilium operator, enforcing NetworkPolicy, and enabling eBPF host routing which could cause short interruption."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="yaml"><pre class="language-yaml"><code class="language-yaml"> bpf<span class="token punctuation">:</span>\n hostLegacyRouting<span class="token punctuation">:</span> <span class="token boolean important">
1false</span>\n cni<span class="token punctuation">:</span>\n customConf<span class="token punctuation">:</span> <span class="token boolean important">false</span>\n operator<span class="token punctuation">:</span>\n unmanagedPodWatcher<span class="token punctuation">:</span>\n restart<span class="token punctuation">:</span> <span class="token boolean important">true</span>\n policyEnforcementMode<span class="token punctuation">:</span> default</code></pre></div>'}}),"\n",i.createElement(a.p,null,"We now have all new nodes only with Cilium as the CNI. We removed Calico resources and Cilium migration labels from the code. As mentioned earlier, we have a daily automatic rotation approach for each worker pool and hence all nodes are replaced gradually and Cilium will be the only CNI available on those nodes."),"\n",i.createElement(a.h2,null,"Conclusion"),"\n",i.createElement(a.p,null,"This blog post walked through how we migrated from Calico to Cilium using the new ",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">CiliumNodeConfig</code>'}})," feature. Overall, we found the process to be smooth and are excited for the possibilities that switching to Cilium will unlock for our platform."),"\n",i.createElement(n,s.A.AmirKheirkhahan))}var r=function(e){void 0===e&&(e={});const{wrapper:a}=Object.assign({},(0,t.RP)(),e.components);return a?i.createElement(a,e,i.createElement(o,e)):o(e)};var l=n(8125),c=n(5805),d=n(8838),p=n(2744);const u=e=>{const{data:{mdx:a},children:n}=e,{frontmatter:{path:t,title:s,date:o,tags:r,ogSummary:d}}=a;return i.createElement(p.A,{headerWithSearch:!0},i.createElement(l.A,{path:t,content:n,date:o,title:s,tags:r,summary:d}),i.createElement(c.A,{className:"my-10 md:my-20 lg:my-28"}))},m=e=>{var a,n;let{data:{mdx:t,site:s},location:{pathname:o}}=e;const{frontmatter:{title:r,ogImage:l,ogSummary:c,dateIso:p,tags:u,author:m}}=t,{siteUrl:g}=s.siteMetadata,h=`${c.slice(0,133)}...`,b=`${g}${o}`,f=null!=l&&null!==(a=l.childImageSharp)&&void 0!==a&&null!==(n=a.resize)&&void 0!==n&&n.src?`${g}${l.childImageSharp.resize.src}`:null,w={title:r,description:h,image:l||null,slug:o},y={"@context":"https://schema.org","@type":"BlogPosting",headline:r,description:h,url:b,datePublished:p,dateModified:p,author:m?{"@type":"Person",name:m}:{"@type":"Organization",name:"Cilium",url:g},publisher:{"@type":"Organization",name:"Cilium",url:g,logo:{"@type":"ImageObject",url:`${g}/images/social-preview.jpg`}},...f&&{image:{"@type":"ImageObject",url:f,width:1200,height:630}},...(null==u?void 0:u.length)>0&&{keywords:u.join(", ")}};return i.createElement(d.A,{data:w,type:"article",datePublished:p,jsonLd:y})};function g(e){return i.createElement(u,e,i.createElement(r,e))}}}]); 2//# sourceMappingURL=component---src-templates-blog-post-jsx-content-file-path-src-posts-2023-08-28-shekner-index-md-9aafd6ad23a24c2d7eab.js.map
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.