1"use strict";(self.webpackChunkcilium_io=self.webpackChunkcilium_io||[]).push([[1262],{6033:function(e,a,n){n.r(a),n.d(a,{Head:function(){return p},default:function(){return b}});var t=n(8453),i=n(6540),s=n(6452);function r(e){const a=Object.assign({img:"img",p:"p",em:"em",br:"br",h1:"h1",a:"a",h2:"h2",span:"span",ol:"ol",li:"li",ul:"ul"},(0,t.RP)(),e.components),{BlogAuthor:n}=a;return n||function(e,a){throw new Error("Expected "+(a?"component":"object")+" `"+e+"` to be defined: you likely forgot to import, pass, or provide it.")}("BlogAuthor",!0),i.createElement(i.Fragment,null,i.createElement(a.img,{src:"/fd1e66143f8ecdd81fdfca1f3d3f0e60/uswitchheader.svg",alt:"USwitch logo"}),"\n",i.createElement(a.p,null,i.createElement(a.em,null,"April 12, 2022"),i.createElement(a.br),"\n",i.createElement(a.em,null,"Author: Joseph Irving, Platform Lead at RVU (Uswitch)")),"\n",i.createElement("a",{id:"multi-cluster-networking-with-cilium-and-friends"}),"\n",i.createElement(a.h1,null,"Multi Cluster Networking with Cilium and Friends"),"\n",i.createElement(a.p,null,"Setting up networking for one Kubernetes cluster can be a challenge but it becomes even more fun once you add multiple clusters into the mix. In this blog, weâll go over the solutions that ",i.createElement(a.a,{href:"https://www.rvu.co.uk/"},"RVU")," (",i.createElement(a.a,{href:"https://www.uswitch.com/"},"Uswitch"),") came up with to allow their applications to talk between clusters and the rationale behind them, from ",i.createElement(a.a,{href:"https://labs.rvu.co.uk/multi-cluster-kubernetes-load-balancing-in-aws-with-yggdrasil-c1583ea7d78f"},"building our own tools")," like ",i.createElement(a.a,{href:"https://github.com/uswitch/yggdrasil"},"Yggdrasil")," (Envoy controller) to implementing other tools such as Cilium. Weâll see what benefits and drawbacks the different approaches can have and also explore why we opted to avoid using a traditional service mesh to achieve our multi-cluster networking goals."),"\n",i.createElement(a.h2,null,"Background"),"\n",i.createElement(a.p,null,"Over the course of around two years, we had a relatively rapid shift in how we thought about infrastructure at our company. Historically, every product team ran their own infrastructure and were almost entirely independent of each other. We realised that teams were duplicating effort and spending far too much time maintaining and building infrastructure instead of developing new features for our websites. To reduce this replication effort, we decided to centralise the infrastructure on a common platform built on top of Kubernetes."),"\n",i.createElement(a.p,null,"As we did this one of the biggest concerns from our teams was: what happens if a cluster has an outage? As these clusters had so much of the website running on them this could be very disruptive. To alleviate these concerns, we built multiple Kubernetes clusters for failover. However, this then created the problem of how do we route traffic between them?"),"\n",i.createElement(a.h2,null,"Building our own tool"),"\n",i.createElement(a.p,null,"Our first approach centred around the open source proxy called ",i.createElement(a.a,{href:"https://www.envoyproxy.io/"},"Envoy"),", its ability to dynamically change its configuration via GRPC was of particular interest to us as we saw a way to leverage our existing infrastructure to configure it. Our idea was to configure Envoy to send traffic to our Kubernetes clusters based on the Ingress objects that were already present in the clusters. We built an Envoy control plane called Yggdrasil. It takes Ingress resources across multiple Kubernetes clusters and turns them into Envoy configuration."),"\n",i.createElement(a.p,null,"If your ingress is in one cluster, Envoy will be configured to send traffic for that host into that cluster."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 993px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/852b3a8cd1c98f0fb06a17cad70997c0/6d6fa/Yggdrasil1.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 55.55555555555556%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/852b3a8cd1c98f0fb06a17cad70997c0/2ff5b/Yggdrasil1.webp 252w,\n/static/852b3a8cd1c98f0fb06a17cad70997c0/4d583/Yggdrasil1.webp 504w,\n/static/852b3a8cd1c98f0fb06a17cad70997c0/a2e27/Yggdrasil1.webp 993w"\n sizes="(max-width: 993px) 100vw, 993px"\n type="image/webp"\n />\n <source\n srcset="/static/852b3a8cd1c98f0fb06a17cad70997c0/019e0/Yggdrasil1.png 252w,\n/static/852b3a8cd1c98f0fb06a17cad70997c0/0dcb2/Yggdrasil1.png 504w,\n/static/852b3a8cd1c98f0fb06a17cad70997c0/6d6fa/Yggdrasil1.png 993w"\n sizes="(max-width: 993px) 100vw, 993px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/852b3a8cd1c98f0fb06a17cad70997c0/6d6fa/Yggdrasil1.png"\n alt="Envoy load balancing to one cluster"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.p,null,"However, if the ingress is in multiple cluster
1s, Envoy will be configured to load balance across them."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 993px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/c46a69508a91938117eae2a7158e0f61/6d6fa/Yggdrasil2.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 55.55555555555556%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/c46a69508a91938117eae2a7158e0f61/2ff5b/Yggdrasil2.webp 252w,\n/static/c46a69508a91938117eae2a7158e0f61/4d583/Yggdrasil2.webp 504w,\n/static/c46a69508a91938117eae2a7158e0f61/a2e27/Yggdrasil2.webp 993w"\n sizes="(max-width: 993px) 100vw, 993px"\n type="image/webp"\n />\n <source\n srcset="/static/c46a69508a91938117eae2a7158e0f61/019e0/Yggdrasil2.png 252w,\n/static/c46a69508a91938117eae2a7158e0f61/0dcb2/Yggdrasil2.png 504w,\n/static/c46a69508a91938117eae2a7158e0f61/6d6fa/Yggdrasil2.png 993w"\n sizes="(max-width: 993px) 100vw, 993px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/c46a69508a91938117eae2a7158e0f61/6d6fa/Yggdrasil2.png"\n alt="Envoy load balancing to multiple clusters"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.p,null,"This allowed us to set up HA services spread across multiple clusters and reduce the risk of outages caused by something going wrong in one cluster. ",i.createElement(a.br),"\n","While this system was primarily intended for user traffic, it inadvertently became a very convenient way for applications to talk to each other. As Envoy would always send traffic to the correct cluster, applications could just talk to Envoy if they wanted to reach another internal service regardless of whether it was running in the same cluster or not."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 960px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/948842c54cdebaf76f84fdc19d8e07b4/7d769/Yggdrasil3.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 75%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/948842c54cdebaf76f84fdc19d8e07b4/2ff5b/Yggdrasil3.webp 252w,\n/static/948842c54cdebaf76f84fdc19d8e07b4/4d583/Yggdrasil3.webp 504w,\n/static/948842c54cdebaf76f84fdc19d8e07b4/10c02/Yggdrasil3.webp 960w"\n sizes="(max-width: 960px) 100vw, 960px"\n type="image/webp"\n />\n <source\n srcset="/static/948842c54cdebaf76f84fdc19d8e07b4/019e0/Yggdrasil3.png 252w,\n/static/948842c54cdebaf76f84fdc19d8e07b4/0dcb2/Yggdrasil3.png 504w,\n/static/948842c54cdebaf76f84fdc19d8e07b4/7d769/Yggdrasil3.png 960w"\n sizes="(max-width: 960px) 100vw, 960px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/948842c54cdebaf76f84fdc19d8e07b4/7d769/Yggdrasil3.png"\n alt="Envoy long request path"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.p,null,"There were a few major drawbacks with this approach. First, a user would talk to one service which then called another service in another cluster by going through Envoy. This resulted in the request having to go out of the cluster and back again adding a fair amount of latency (P95 of around 20ms per trip). This round trip also caused us to lose identity information. As far as our apps were concerned everything was coming from Envoy."),"\n",i.createElement(a.p,null,"This kind of cross-cluster functionality was clearly useful to our development teams, however the implementation was less than ideal. We concluded that we would need something more like a service mesh for service to service communication to work in a sensible way."),"\n",i.createElement("a",{id:"searching-for-a-service-mesh"}),"\n",i.createElement(a.h1,null,"Searching for a Service Mesh"),"\n",i.createElement(a.p,null,"When evaluating the various different service meshes that existed we started with three main requirements:"),"\n",i.createElement(a.ol,null,"\n",i.createElement(a.li,null,"Multi-cluster services - this was the main one, we wanted a way to talk between clusters, service to service."),"\n",i.createElement(a.li,null,"âRealâ Pod IPs - meaning that the pods get an IP address that belongs to the VPC theyâre running in, instead of being assigned a virtual IP address. This was âa nice to haveâ as it can simplify networking complexity and allow for more VPC native tools like flow logs, security groups, etc to work with pods easily."),"\n",i.createElement(a.li,null,"Easy to implement on our existing stack - whatever we chose weâd rather it didnât require significant reworks to how our existing applications work."),"\n"),"\n",i.createElement(a.p,null,"With these in mind we began our testing and comparison on the various meshes around at the time and as we did, a few common problems were found."),"\n",i.createElement(a.p,null,"One of the biggest drawbacks was a reliance on sidecars as the main mechanism for implementing the mesh. This caused all sorts of complications including:"),"\n",i.createElement(a.ul,null,"\n",i.createElement(a.li,null,"Service mesh is unavailable during the init phase - there are no sidecars in the init phase so the mesh functionality does not work."),"\n",i.createElement(a.li,null,"Job pods get stuck in a state where they never complete - the sidecars need some kind of logic to understand that theyâre in a job and should shut down once the primary container has finished its job."),"\n",i.createElement(a.li,null,"Startup/Shutdown ordering can be an issue - if the sidecar starts up/shuts down after/before your containers you can get networking issues"),"\n"),"\n",i.createElement(a.p,null,"While all of these problems do have solutions, it still seemed like a lot of work compared to what we wanted to achieve."),"\n",i.createElement(a.p,null,"After looking at more âtraditionalâ service meshes, we discovered Cilium which appeared to meet our requirements very well:"),"\n",i.createElement(a.ul,null,"\n",i.createElement(a.li,null,"Run as a daemonset - no sidecars required"),"\n",i.createElement(a.li,null,"Support ârealâ IPs for Pods using AWS ENIs"),"\n",i.createElement(a.li,null,"No application changes needed to leverage its capabilities - everything is done at the host level and is transparent to the applications using it."),"\n"),"\n",i.createElement(a.p,null,"This all sounded great, so the next step was to try it out!"),"\n",i.createElement("a",{id:"cilium-time"}),"\n",i.createElement(a.h2,null,"Cilium Time"),"\n",i.createElement(a.p,null,"So let's compare Cilium to our original requirements, first real pod IPs:"),"\n",i.createElement(a.p,null,"Cilium does this in AWS by associating ENIs with your instance that it can then assign additional IPs to, each IP corresponding to one of the pods on the node. This means every pod has an IP that is a valid IP in the VPC youâre running in, allowing all the normal AWS networking features to work with them. No virtual network needed!"),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/52ab3/Cilium1.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 55.55555555555556%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/2ff5b/Cilium1.webp 252w,\n/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/4d583/Cilium1.webp 504w,\n/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/905a7/Cilium1.webp 1008w,\n/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/bb9f8/Cilium1.webp 1512w,\n/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/550a2/Cilium1.webp 1564w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/019e0/Cilium1.png 252w,\n/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/0dcb2/Cilium1.png 504w,\n/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/832a9/Cilium1.png 1008w,\n/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/19357/Cilium1.png 1512w,\n/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/52ab3/Cilium1.png 1564w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/807bfba65d5d9ecc5f4b4ec9d8bcb4d8/832a9/Cilium1.png"\n alt="Cilium assigning pod IPs"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.p,null,"Cilium maps pod IPs to services in the same way a more traditional setup would work, but instead of just relying on IPTables rules, Cilium can replace kube-proxy leveraging eBPF which at scale outperforms IPTables and ensures Service Endpoint changes are atomic using eBPF maps."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/bc7e5dbf85f95a455cf7833a21f183fb/0d4f8/Cilium2.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 50.39682539682539%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/bc7e5dbf85f95a455cf7833a21f183fb/2ff5b/Cilium2.webp 252w,\n/static/bc7e5dbf85f95a455cf7833a21f183fb/4d583/Cilium2.webp 504w,\n/static/bc7e5dbf85f95a455cf7833a21f183fb/905a7/Cilium2.webp 1008w,\n/static/bc7e5dbf85f95a455cf7833a21f183fb/bb9f8/Cilium2.webp 1512w,\n/static/bc7e5dbf85f95a455cf7833a21f183fb/485a2/Cilium2.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/bc7e5dbf85f95a455cf7833a21f183fb/019e0/Cilium2.png 252w,\n/static/bc7e5dbf85f95a455cf7833a21f183fb/0dcb2/Cilium2.png 504w,\n/static/bc7e5dbf85f95a455cf7833a21f183fb/832a9/Cilium2.png 1008w,\n/static/bc7e5dbf85f95a455cf7833a21f183fb/19357/Cilium2.png 1512w,\n/static/bc7e5dbf85f95a455cf7833a21f183fb/0d4f8/Cilium2.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/bc7e5dbf85f95a455cf7833a21f183fb/832a9/Cilium2.png"\n alt="Cilium eBPF rules"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.p,null,"The (Cluster) Mesh works by allowing Cilium to populate Service Endpoints in each Cluster using eBPF Maps. If you have the same service in two different cluster
1s, Cilium will combine all those pod IPs as possible destinations. Thus, when you talk to the service address, you will get sent to any cluster where that service has endpoints."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/c60e5dd6d6704658a57f137ae4ba5e10/e8a46/Cilium3.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 32.93650793650794%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/c60e5dd6d6704658a57f137ae4ba5e10/2ff5b/Cilium3.webp 252w,\n/static/c60e5dd6d6704658a57f137ae4ba5e10/4d583/Cilium3.webp 504w,\n/static/c60e5dd6d6704658a57f137ae4ba5e10/905a7/Cilium3.webp 1008w,\n/static/c60e5dd6d6704658a57f137ae4ba5e10/58f09/Cilium3.webp 1282w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/c60e5dd6d6704658a57f137ae4ba5e10/019e0/Cilium3.png 252w,\n/static/c60e5dd6d6704658a57f137ae4ba5e10/0dcb2/Cilium3.png 504w,\n/static/c60e5dd6d6704658a57f137ae4ba5e10/832a9/Cilium3.png 1008w,\n/static/c60e5dd6d6704658a57f137ae4ba5e10/e8a46/Cilium3.png 1282w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/c60e5dd6d6704658a57f137ae4ba5e10/832a9/Cilium3.png"\n alt="Cilium Service Endpoints across clusters"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.p,null,"This makes the mesh completely transparent to normal applications, they just need to talk to the service in their cluster as they normally would and theyâll be talking across Kubernetes clusters!"),"\n",i.createElement(a.p,null,'Establishing load-balancing between clusters is achieved by defining a Kubernetes service with identical name and namespace in each cluster and adding the annotation io.cilium/global-service: "true" to declare it global. Cilium will automatically perform load-balancing to pods in both clusters.'),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/da5a270708facd1f7f780b807f4597b6/63f34/Cilium4.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 86.5079365079365%; position: relative; bottom: 0; left: 0; background-image: url(\'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAARCAYAAADdRIy+AAAACXBIWXMAAAsTAAALEwEAmpwYAAACi0lEQVR42o1UWXabQBDURWJbG8wCM+zDKpDB2izJS5yvJPc/RqVBzrNsx3Y+6o0AUd1dVc3o/uERYZIgLQqYPD+dRY4sL5Bm2XB+hbwoEYQRfv76jVFTL5FPBGrbw8JSMFOJYuaCMYk4MZhb9pewGce3i0vsD0eMVt0KGzvA3jFIvBjbbo2ubNAsW5TVAowLCOl8CsdVmExn6Kcdde0N1paPvTTonBjruIT2AvhBCC4cWDYbOuBCfgjpuBhPpkT4gNF12yG0JBEmuOMGqYqQlaRJFJOGORKT0ujpQPwXfdcfEi7bFjPLGggPIkGlYtxmNYokx/XNCk2zxHa3w6KuUZEEvQxKe69I3xEyi+FII/eED8Jgx6LBpNTk8OIYMbmfVxVhgZS6d7SGxZ9lIDkkkY/HkxfCKTmVSx8rEaEVIToZDeQtD2GYQmXrQZaAENsuEq6QCh9cabAshagKXDkS93f3Jw0j+mMkvQFauPCFQiY81CJALYOhQEYF+6IlIRMaBd3nkuIVkXnXDcaMvRDWlqaRUwQ9aU/IHExJ/IltwxICM4L1jPnZ72FkGlfS2GcjdxiTKQVVPpAxHY25MWRKsUBETvcvCYrP6TzhXWzcN7GxSMNAajyJlMYKEZQ51tst2tVqcPNfUfk0Npxc3lJ3JWlWOCGULeHOGeSM0xboYV/fRuXTYPf7OJMSirq8Jad35PBWJdjHBsIlc5oGmjZnbvP/J+REqEmrI3VaUXQMrd/j0sDhzpC1yHcReuprwpo2oV9si15i1MGCAn1jUyYJyxmFm649pbCpKEJGv1rB81W8uLzC4XiH0XqzHSp4fkAfBR+K0J9uD/90Ks+Dq0/Qz8/P0X9IetLvTz/wB65pGmf9DnjpAAAAAElFTkSuQmCC\'); background-size: cover; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/da5a270708facd1f7f780b807f4597b6/2ff5b/Cilium4.webp 252w,\n/static/da5a270708facd1f7f780b807f4597b6/4d
1583/Cilium4.webp 504w,\n/static/da5a270708facd1f7f780b807f4597b6/905a7/Cilium4.webp 1008w,\n/static/da5a270708facd1f7f780b807f4597b6/3f888/Cilium4.webp 1384w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/da5a270708facd1f7f780b807f4597b6/019e0/Cilium4.png 252w,\n/static/da5a270708facd1f7f780b807f4597b6/0dcb2/Cilium4.png 504w,\n/static/da5a270708facd1f7f780b807f4597b6/832a9/Cilium4.png 1008w,\n/static/da5a270708facd1f7f780b807f4597b6/63f34/Cilium4.png 1384w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/da5a270708facd1f7f780b807f4597b6/832a9/Cilium4.png"\n alt="Cilium YAML file"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.p,null,"Our new cross cluster journey for applications is just one hop. From one pod to another pod via their pod IP. Not only does this remove a load of latency but it also maintains identity so we can now use things like Network Policy to control what services can talk to each other."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 960px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/187768ac51b277ffd1a3122de5efc8db/7d769/Cilium5.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 75%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/187768ac51b277ffd1a3122de5efc8db/2ff5b/Cilium5.webp 252w,\n/static/187768ac51b277ffd1a3122de5efc8db/4d583/Cilium5.webp 504w,\n/static/187768ac51b277ffd1a3122de5efc8db/10c02/Cilium5.webp 960w"\n sizes="(max-width: 960px) 100vw, 960px"\n type="image/webp"\n />\n <source\n srcset="/static/187768ac51b277ffd1a3122de5efc8db/019e0/Cilium5.png 252w,\n/static/187768ac51b277ffd1a3122de5efc8db/0dcb2/Cilium5.png 504w,\n/static/187768ac51b277ffd1a3122de5efc8db/7d769/Cilium5.png 960w"\n sizes="(max-width: 960px) 100vw, 960px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/187768ac51b277ffd1a3122de5efc8db/7d769/Cilium5.png"\n alt="Cilium direct routing"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.p,null,"The actual networking setup for this was also quite simple thanks to the use of AWS Transit Gateway which we use to peer all our clusters together. Since all our pod IPs were ârealâ, as long as routes and Security groups were set up, it all just worked seamlessly."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/5ddad/AWS1.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 53.17460317460318%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/2ff5b/AWS1.webp 252w,\n/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/4d583/AWS1.webp 504w,\n/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/905a7/AWS1.webp 1008w,\n/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/bb9f8/AWS1.webp 1512w,\n/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/1d82c/AWS1.webp 1588w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/019e0
1/AWS1.png 252w,\n/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/0dcb2/AWS1.png 504w,\n/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/832a9/AWS1.png 1008w,\n/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/19357/AWS1.png 1512w,\n/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/5ddad/AWS1.png 1588w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/8b2cfb1f9821e77532d10b7c4e3d5a1d/832a9/AWS1.png"\n alt="AWS transit gateway connecting clusters"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.p,null,"We even extended this to go across clouds by setting up a VPN between Google Cloud and AWS. Our GKE and EKS clusters talk to each other via the Cilium Clustermesh without any hassle."),"\n",i.createElement(a.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/b040b4d9cb9b1556afcf69bc589d353f/0d4f8/multicloud1.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 46.42857142857143%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/b040b4d9cb9b1556afcf69bc589d353f/2ff5b/multicloud1.webp 252w,\n/static/b040b4d9cb9b1556afcf69bc589d353f/4d583/multicloud1.webp 504w,\n/static/b040b4d9cb9b1556afcf69bc589d353f/905a7/multicloud1.webp 1008w,\n/static/b040b4d9cb9b1556afcf69bc589d353f/bb9f8/multicloud1.webp 1512w,\n/static/b040b4d9cb9b1556afcf69bc589d353f/485a2/multicloud1.webp 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/b040b4d9cb9b1556afcf69bc589d353f/019e0/multicloud1.png 252w,\n/static/b040b4d9cb9b1556afcf69bc589d353f/0dcb2/multicloud1.png 504w,\n/static/b040b4d9cb9b1556afcf69bc589d353f/832a9/multicloud1.png 1008w,\n/static/b040b4d9cb9b1556afcf69bc589d353f/19357/multicloud1.png 1512w,\n/static/b040b4d9cb9b1556afcf69bc589d353f/0d4f8/multicloud1.png 1600w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/b040b4d9cb9b1556afcf69bc589d353f/832a9/multicloud1.png"\n alt="VPN from Google Cloud to AWS"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",i.createElement(a.h2,null,"Whatâs Next?"),"\n",i.createElement(a.p,null,"Cilium proved to be a great option for us, giving us the functionality we wanted without all the complication that the more traditional service meshes typically imposed on a user. However, it wasnât completely without its drawbacks. The lack of a dedicated proxy in Cilium meant that apps had to implement things like retries and load-balancing when talking to other services (something our old setup or going through Envoy did for them). Another thing to be aware of is that Cilium differs from more traditional IPTables based implementations, so it is a good idea to familiarise yourself with how Cilium operates and how it uses eBPF maps. The Cilium community was always very helpful if we did ever find any bugs though, so rest assured someone will help you!"),"\n",i.createElement(a.p,null,"Itâs also worth noting that Cilium have now started the beta of their Cilium based service mesh which promises to add some more service-meshy features like retries and canary deployments, but with the same transparent sidecar-less approach that vanilla Cilium provides. Weâll be watching this one closely!"),"\n",i.createElement(n,s.A.JosephIrving))}var o=function(e){void 0===e&&(e={});const{wrapper:a}=Object.assign({},(0,t.RP)(),e.components);return a?i.createElement(a,e,i.createElement(r,e)):r(e)};var l=n(8125),c=n(5805),d=n(8838),u=n(2744);const m=e=>{const{data:{mdx:a},children:n}=e,{frontmatter:{path:t,title:s,date:r,tags:o,ogSummary:d}}=a;return i.createElement(u.A,{headerWithSearch:!0},i.createElement(l.A,{path:t,content:n,date:r,title:s,tags:o,summary:d}),i.createElement(c.A,{className:"my-10 md:my-20 lg:my-28"}))},p=e=>{var a,n;let{data:{mdx:t,site:s},location:{pathname:r}}=e;const{frontmatter:{title:o,ogImage:l,ogSummary:c,dateIso:u,tags:m,author:p}}=t,{siteUrl:b}=s.siteMetadata,g=`${c.slice(0,133)}...`,h=`${b}${r}`,f=null!=l&&null!==(a=l.childImageSharp)&&void 0!==a&&null!==(n=a.resize)&&void 0!==n&&n.src?`${b}${l.childImageSharp.resize.src}`:null,w={title:o,description:g,image:l||null,slug:r},y={"@context":"https://schema.org","@type":"BlogPosting",headline:o,description:g,url:h,datePublished:u,dateModified:u,author:p?{"@type":"Person",name:p}:{"@type":"Organization",name:"Cilium",url:b},publisher:{"@type":"Organization",name:"Cilium",url:b,logo:{"@type":"ImageObject",url:`${b}/images/social-preview.jpg`}},...f&&{image:{"@type":"ImageObject",url:f,width:1200,height:630}},...(null==m?void 0:m.length)>0&&{keywords:m.join(", ")}};return i.createElement(d.A,{data:w,type:"article",datePublished:u,jsonLd:y})};function b(e){return i.createElement(m,e,i.createElement(o,e))}},6452:function(e,a){a.A={thomasGraf:{header:"Thomas Graf",bio:'Thomas Graf is a Co-Founder of Cilium and the CTO & Co-Founder of <a href="https://isovalent.com/?utm_source=website-cilium&utm_medium=referral&utm_campaign=cilium-enterprise">Isovalent</a>, the company behind Cilium. Before that, Thomas spent 15 years as\n a kernel developer working on the <a href="https://kernel.org">Linux kernel</a> in networking, security and eventually eBPF.'},lizRice:{header:'<a href="https://twitter.com/lizrice">Liz Rice</a>',bio:'Liz is Chief Open Source Officer at <a href="https://isovalent.com/?utm_source=website-cilium&utm_medium=referral&utm_campaign=cilium-enterprise" target="_blank" rel="noopener noreferrer">Isovalent</a>, the company behind Cilium. She is also chair of the CNCF\'s Technical Oversight Committee, and the author of Container Security published by O\'Reilly.'},luanGuimaraes:{header:"Luan Guimarães",bio:"Luan is a Brazilian rock climber, amateur musician, and\n programmer and am enthusiastic about free software communities and other\n open knowledge initiatives. He has been working as a Site Reliability\n Engineer at Wildlife Studios, using and building infrastru
1cture tools on\n top of Kubernetes in order to support millions of users around the world."},joshVanLeeuwen:{header:"Josh Van Leeuwen",bio:"Josh interned at Jetstack during the summer of 2017 before continuing to\n work part time during his final year of study at the University of Bristol.\n During this year, Josh developed a Kubernetes custom controller that\n automates the delegation of RBAC permissions based on time and event\n triggers. This work was later awarded the best Software Development Tool\n Final Year Project. Josh now works full time at Jetstack where if heâs not\n writing more Go, heâs making good food."},howardHao:{header:"Howard Hao",bio:" Howard Hao has been working as a Site Reliability Engineer for five years at\n Ect888.com since graduating from Shanghai Jiao Tong University. His team\n consists of 7 members and has been focusing on the construction of\n container orchestration platform like Kubernetes for one and a half years."},sergeyGeneralov:{header:"Sergey Generalov",bio:"Sergey is a member of the technical staff at Isovalent\n and focuses on helping Cilium users solve challenges related\n to network policies, monitoring, and connectivity troubleshooting\n by building tools like Network Policy Editor, Hubble UI and more."},liWenquan:{header:"Li Wenquan",bio:"Hello everyone, I am Li Wenquan from China. You can call me David. I\n started my Docker journey from 2014 and now work as a project manager of\n enterprise container platform, which is built on Kubernetes and Mesos. I\n got to know Cilium project from Kubecon, it is so interesting and\n promising. I've learned a lot from it, such as BPF, XDP and how to replace\n kube-proxy in a elegant way and I'd love to contribute to it."},alexanderAlemayhu:{header:"Alexander Alemayhu",bio:"Alexander Alemayhu is a software engineer at Isovalent,\n the company behind Cilium. He has been working on eBPF and Linux\n kernel technologies for several years, focusing on networking and observability solutions."},DanielBorkmann:{header:"Daniel Borkmann",bio:"Daniel Borkmann is a Distinguished Software Engineer, Isovalent at Cisco"},ThomasGraf:{header:"Thomas Graf",bio:"Thomas Graf is the CTO & Co-Founder Isovalent and also the Vice President Security Cisco"},JedSalazar:{header:"Jed Salazar",bio:"Jed Salazar is a Senior Solutions Architect, Isovalent"},JedSalazarandJoeStringer:{header:"Jed Salazar and Joe Stringer",bio:"Jed Salazar is a Senior Solutions Architect at Isovalent\n and Joe Stringer is a Principal Engineer, Isovalent at Cisco"},JosephIrving:{header:"Joseph Irving",bio:"Joseph Irving is a Platform Engineer Lead at RVU (Uswitch)"},BillMulligan:{header:"Bill Mulligan",bio:"Bill Mulligan is a Cilium and eBPF Community Pollinator,\n Isovalent at Cisco and a Governing Board Member of the eBPF Foundation."},OndrejBlazek:{header:"Ondrej Blazek",bio:"Ondrej Blazek is an Infrastructure Engineer at Seznam.cz"},LeonardCohnenandMoritzEckert:{header:"Leonard Cohnen and Moritz Eckert",bio:"Leonard Cohnen and Moritz Eckert are team members at Edgeless Systems"},PolArroyo:{header:"Pol Arroyo",bio:"Pol Arroyo is a DevOps Engineer at Hetzner Cloud."},JedSalazarandMartynasPumputis:{header:"Jed Salazar and Martynas Pumputis",bio:"Jed Salazar is a Senior Solutions Architect, Isovalent and Martynas Pumputis is a Principal Software Engineer, Isovalent at Cisco"},ShedrackAkintayo:{header:"Shedrack Akintayo",bio:"Shedrack Akintayo is a Community Manager at\n Isovalent helping build the eBPF and Cilium open source communities"},AmirKheirkhahan:{header:"Amir Kheirkhahan",bio:"Amir Kheirkhahan is a DevOps Specialist at DB Schenker handling design, development,\n deployment and maintenance of wide range of devops toolchain on top of Kubernetes clusters"},PaulArah:{header:"Paul Arah",bio:"Paul Arah is a Community Builder focused on Security at Isovalent (Cisco)"},HimalKumar:{header:"Himal Kumar, Bhaskar Dutta, Arman Pashamokhtari",bio:"Himal Kumar, Bhaskar Dutta, Arman Pashamokhtari are all part of the\n CanopusAI team Real Time Network Observability, powered by eBPF and Agentic AI"},DoniaChaiehloudj:{header:"Donia Chaiehloudj",bio:"Donia Chaiehloudj is a Senior Softw
1are Engineer and Community Oriented at Isovalent.\n She has been working on Cilium and eBPF technologies, focusing on networking and security solutions."},KatieMeinders:{header:"Katie Meinders",bio:"Katie Meinders is a Community Builder at Isovalent where\n she helps grow the Cilium and eBPF communities through storytelling,\n social media, showcasing user success, and building connections across the open source ecosystem."},PeaceSandy:{header:"Peace Sandy",bio:"Peace Sandy is an LFX mentee who contributed to improving Cilium SEO, AEO, and\n AIO during her mentorship period."},NehaAggarwal:{header:"Neha Aggarwal",bio:"Neha Aggarwal is a Principal Engineer at Microsoft."},CharityMbisi:{header:"Charity Mbisi",bio:"Charity Mbisi is an LFX mentee who contributed to improving Cilium's SEO, AEO, and AIO during his mentorship period.\n Professionally, Charity Mbisi is a Software Engineer consulting in the Fin-tech and banking industry, specializing in building cloud native computing solutions and optimized service delivery."},andreMartinsAndFerozSalam:{header:"André Martins and Feroz Salam",bio:"André Martins is a Cilium maintainer and Software Engineer, Isovalent at Cisco.\n Feroz Salam is a member of the Cilium Security Team and a Security Engineer, Isovalent at Cisco."},ChristianHernandez:{header:"Christian Hernandez",bio:"Christian is a well rounded technologist with experience in infrastructure engineering, systems administration, enterprise architecture, tech support, advocacy, and product management. Passionate about OpenSource and containerizing the world one application at a time. He is currently a maintainer of the Argo Project and OpenGitops. Currently, he works as a Technical Marketing Engineer and Tech Lead at Cisco. He focuses on GitOps practices, DevOps, Kubernetes, Network Security, and Containers."},AkilaInduranga:{header:"Akila Induranga",bio:'Akila is a Senior Software Engineer at WSO2, and a maintainer of <a href="https://openchoreo.dev/" target="_blank" rel="noopener noreferrer">OpenChoreo</a>, an open-source internal developer platform for Kubernetes and a CNCF sandbox project.\n He works on the platform\'s observability and networking layers, including the Cilium-based networking module that brings identity-based policy and Hubble observability to OpenChoreo cells.'}}}}]); 2//# sourceMappingURL=component---src-templates-blog-post-jsx-content-file-path-src-posts-2022-04-12-multi-cluster-networking-index-md-4112fc92319b26324173.js.map
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.