1"use strict";(self.webpackChunkk_3_s_docs=self.webpackChunkk_3_s_docs||[]).push([[7431],{25548:(e,n,s)=>{s.r(n),s.d(n,{assets:()=>o,contentTitle:()=>i,default:()=>h,frontMatter:()=>l,metadata:()=>r,toc:()=>c});const r=JSON.parse('{"id":"datastore/cluster-loadbalancer","title":"Cluster Load Balancer","description":"This section describes how to install an external load balancer in front of a High Availability (HA) K3s cluster\'s server nodes. Two examples are provided: Nginx and HAProxy.","source":"@site/docs/datastore/cluster-loadbalancer.md","sourceDirName":"datastore","slug":"/datastore/cluster-loadbalancer","permalink":"/datastore/cluster-loadbalancer","draft":false,"unlisted":false,"editUrl":"https://github.com/k3s-io/docs/edit/main/docs/datastore/cluster-loadbalancer.md","tags":[],"version":"current","lastUpdatedAt":1789781277000,"frontMatter":{"title":"Cluster Load Balancer"},"sidebar":"mySidebar","previous":{"title":"High Availability External DB","permalink":"/datastore/ha"},"next":{"title":"Upgrades","permalink":"/upgrades/"}}');var a=s(74848),t=s(28453);const l={title:"Cluster Load Balancer"},i=void 0,o={},c=[{value:"Prerequisites",id:"prerequisites",level:2},{value:"Setup Load Balancer",id:"setup-load-balancer",level:2},{value:"Nginx Load Balancer",id:"nginx-load-balancer",level:2},{value:"Kube-VIP",id:"kube-vip",level:2}];function d(e){const n={a:"a",admonition:"admonition",code:"code",h2:"h2",li:"li",ol:"ol",p:"p",pre:"pre",ul:"ul",...(0,t.R)(),...e.components},{TabItem:s,Tabs:r}=n;return s||p("TabItem",!0),r||p("Tabs",!0),(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.p,{children:"This section describes how to install an external load balancer in front of a High Availability (HA) K3s cluster's server nodes. Two examples are provided: Nginx and HAProxy."}),"\n",(0,a.jsxs)(n.admonition,{type:"tip",children:[(0,a.jsxs)(n.p,{children:["External load-balancers should not be confused with the embedded ServiceLB, which is an embedded controller that allows for use of Kubernetes LoadBalancer Services without deploying a third-party load-balancer controller. For more details, see ",(0,a.jsx)(n.a,{href:"/networking/networking-services#service-load-balancer",children:"Service Load Balancer"}),"."]}),(0,a.jsx)(n.p,{children:"External load-balancers can be used to provide a fixed registration address for registering nodes, or for external access to the Kubernetes API Server. For exposing LoadBalancer Services, external load-balancers can be used alongside or instead of ServiceLB, but in most cases, replacement load-balancer controllers such as MetalLB or Kube-VIP are a better choice."})]}),"\n",(0,a.jsx)(n.h2,{id:"prerequisites",children:"Prerequisites"}),"\n",(0,a.jsx)(n.p,{children:"All nodes in this example are running Ubuntu 20.04."}),"\n",(0,a.jsxs)(n.p,{children:["For both examples, assume that a ",(0,a.jsx)(n.a,{href:"/datastore/ha-embedded",children:"HA K3s cluster with embedded etcd"})," has been installed on 3 nodes."]}),"\n",(0,a.jsx)(n.p,{children:"Each k3s server is configured with:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yaml",children:"# /etc/rancher/k3s/config.yaml\ntoken: lb-cluster-gd\ntls-san: 10.10.10.100\n"})}),"\n",(0,a.jsx)(n.p,{children:"The nodes have hostnames and IPs of:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["server-1: ",(0,a.jsx)(n.code,{children:"10.10.10.50"})]}),"\n",(0,a.jsxs)(n.li,{children:["server-2: ",(0,a.jsx)(n.code,{children:"10.10.10.51"})]}),"\n",(0,a.jsxs)(n.li,{children:["server-3: ",(0,a.jsx)(n.code,{children:"10.10.10.52"})]}),"\n"]}),"\n",(0,a.jsx)(n.p,{children:"Two additional nodes for load balancing are configured with hostnames and IPs of:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["lb-1: ",(0,a.jsx)(n.code,{children:"10.10.10.98"})]}),"\n",(0,a.jsxs)(n.li,{children:["lb-2: ",(0,a.jsx)(n.code,{children:"10.10.10.99"})]}),"\n"]}),"\n",(0,a.jsx)(n.p,{children:"Three additional nodes exist with hostnames and IPs of:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["agent-1: ",(0,a.jsx)(n.code,{children:"10.10.10.101"})]}),"\n",(0,a.jsxs)(n.li,{children:["agent-2: ",(0,a.jsx)(n.code,{children:"10.10.10.102"})]}),"\n",(0,a.jsxs)(n.li,{children:["agent-3: ",(0,a.jsx)(n.code,{children:"10.10.10.103"})]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"setup-load-balancer",children:"Setup Load Balancer"}),"\n",(0,a.jsxs)(r,{queryString:"ext-load-balancer",children:[(0,a.jsxs)(s,{value:"HAProxy",default:!0,children:[(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.a,{href:"http://www.haproxy.org/",children:"HAProxy"})," is an open source option that provides a TCP load balancer. It also supports HA for the load balancer itself, ensuring redundancy at all levels. See ",(0,a.jsx)(n.a,{href:"http://docs.haproxy.org/2.8/intro.html",children:"HAProxy Documentation"})," for more info."]}),(0,a.jsxs)(n.p,{children:["Additionally, we will use KeepAlived to generate a virtual IP (VIP) that will be used to access the cluster. See ",(0,a.jsx)(n.a,{href:"https://www.keepalived.org/documentation/",children:"KeepAlived Documentation"})," for more info."]}),(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsx)(n.li,{children:"Install HAProxy and KeepAlived:"}),"\n"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"sudo apt-get install haproxy keepalived\n"})}),(0,a.jsxs)(n.ol,{start:"2",children:["\n",(0,a.jsxs)(n.li,{children:["Add the following to ",(0,a.jsx)(n.code,{children:"/etc/haproxy/haproxy.cfg"})," on lb-1 and lb-2:"]}),"\n"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"frontend k3s-frontend\n bind *:6443\n mode tcp\n option tcplog\n default_backend k3s-backend\n\nbackend k3s-backend\n mode tcp\n option tcp-check\n balance roundrobin\n default-server inter 10s downinter 5s\n server server-1 10.10.10.50:6443 check\n server server-2 10.10.10.51:6443 check\n server server-3 10.10.10.52:6443 check\n"})}),(0,a.jsxs)(n.ol,{start:"3",children:["\n",(0,a.jsxs)(n.li,{children:["Add the following to ",(0,a.jsx)(n.code,{children:"/etc/keepalived/keepalived.conf"})," on lb-1 and lb-2:"]}),"\n"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"global_defs {\n enable_script_security\n script_user root\n}\n\nvrrp_script chk_haproxy {\n script 'killall -0 haproxy' # faster than pidof\n interval 2\n}\n\nvrrp_instance haproxy-vip {\n interface eth1\n state <STATE> # MASTER on lb-1, BACKUP on lb-2\n priority <PRIORITY> # 200 on lb-1, 100 on lb-2\n\n virtual_router_id 51\n\n virtual_ipaddress {\n 10.10.10.100/24\n }\n\n track_script {\n chk_haproxy\n }\n}\n"})}),(0,a.jsxs)(n.ol,{start:"6",children:["\n",(0,a.jsx)(n.li,{children:"Restart HAProxy and KeepAlived on lb-1 and lb-2:"}),"\n"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"systemctl restart haproxy\nsystemctl restart keepalived\n"})}),(0,a.jsxs)(n.ol,{start:"5",children:["\n",(0,a.jsx)(n.li,{children:"On agent-1, agent-2, and agent-3, run the following command to install k3s and join the cluster:"}),"\n"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"curl -sfL https://get.k3s.io | K3S_TOKEN=lb-cluster-gd sh -s - agent --server https://10.10.10.100:6443\n"})}),(0,a.jsxs)(n.p,{children:["You can now use ",(0,a.jsx)(n.code,{children:"kubectl"})," from server node to interact with the cluster."]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"root@server-1 $ k3s kubectl get nodes -A\nNAME STATUS ROLES AGE VERSION\nagent-1 Ready <none> 32s v1.27.3+k3s1\nagent-2 Ready <none> 20s v1.27.3+k3s1\nagent-3 Ready <none> 9s v1.27.3+k3s1\nserver-1 Ready control-plane,etcd,master 4m22s v1.27.3+k3s1\nserver-2 Ready control-plane,etcd,master 3m58s v1.27.3+k3s1\nserver-3 Ready control-plane,etcd,master 3m12s v1.27.3+k3s1\n"})})]}),(0,a.jsxs)(s,{value:"Nginx",children:[(0,a.jsx)(n.h2,{id:"nginx-load-balancer",children:"Nginx Load Balancer"}),(0,a.jsx)(n.admonition,{type:"danger",children:(0,a.jsx)(n.p,{children:"Nginx does not natively support a High Availability (HA) configuration. If setting up an HA cluster, having a single load balancer in front of K3s will reintroduce a single point of failure."})}),(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.a,{href:"http://nginx.org/",children:"Nginx Open Source"})," provides a TCP load balancer. See ",(0,a.jsx)(n.a,{href:"https://nginx.org/en/docs/http/load_balancing.html",children:"Using nginx as HTTP load balancer"})," for more info."]}),(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["Create a ",(0,a.jsx)(n.code,{children:"nginx.conf"})," file on lb-1 with the following contents:"]}),"\n"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"events {}\n\nstream {\n upstream k3s_servers {\n server 10.10.10.50:6443;\n server 10.10.10.51:6443;\n server 10.10.10.52:6443;\n }\n\n server {\n listen 6443;\n proxy_pass k3s_servers;\n }\n}\n"})}),(0,a.jsxs)(n.ol,{start:"2",children:["\n",(0,a.jsx)(n.li,{children:"Run the Nginx load balancer on lb-1:"}),"\n"]}),(0,a.jsx)(n.p,{children:"Using docker:"}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"docker run -d --restart unless-stopped \\\n -v ${PWD}/nginx.conf:/etc/nginx/nginx.conf \\\n -p 6443:6443 \\\n nginx:stable\n"})}),(0,a.jsxs)(n.p,{children:["Or ",(0,a.jsx)(n.a,{href:"https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-open-source/",children:"install nginx"})," and then run:"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"cp nginx.conf /etc/nginx/nginx.conf\nsystemctl start nginx\n"})}),(0,a.jsxs)(n.ol,{start:"3",children:["\n",(0,a.jsx)(n.li,{children:"On agent-1, agent-2, and agent-3, run the following command to install k3s and join the cluster:"}),"\n"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"curl -sfL https://get.k3s.io | K3S_TOKEN=lb-cluster-gd sh -s - agent --server https://10.10.10.98:6443\n"})}),(0,a.jsxs)(n.p,{children:["You can now use ",(0,a.jsx)(n.code,{children:"kubectl"})," from server node to interact with the cluster."]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"root@server1 $ k3s kubectl get nodes -A\nNAME STATUS ROLES AGE VERSION\nagent-1 Ready <none> 30s v1.27.3+k3s1\nagent-2 Ready <none>
1 22s v1.27.3+k3s1\nagent-3 Ready <none> 13s v1.27.3+k3s1\nserver-1 Ready control-plane,etcd,master 4m49s v1.27.3+k3s1\nserver-2 Ready control-plane,etcd,master 3m58s v1.27.3+k3s1\nserver-3 Ready control-plane,etcd,master 3m16s v1.27.3+k3s1\n"})})]}),(0,a.jsxs)(s,{value:"Kube-VIP",children:[(0,a.jsx)(n.h2,{id:"kube-vip",children:"Kube-VIP"}),(0,a.jsx)(n.admonition,{type:"info",children:(0,a.jsx)(n.p,{children:"This example configures kube-vip in ARP (layer\u20112) mode to provide a Virtual IP (VIP) and a control-plane load balancer. The manifest below deploys kube-vip as a DaemonSet on control-plane nodes and announces the VIP on the node network. Adjust the interface and subnet to match your environment."})}),(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.a,{href:"https://kube-vip.io/",children:"Kube-VIP"})," provides a virtual IP and load balancer for the Kubernetes control plane and for Services of type LoadBalancer. The instructions below show how to generate and deploy the daemonset manifest on K3s control-plane nodes."]}),(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsx)(n.li,{children:"Install the RBAC manifest:"}),"\n"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"curl -fsSL https://kube-vip.io/manifests/rbac.yaml -o /var/lib/rancher/k3s/server/manifests/kube-vip-rbac.yaml\n"})}),(0,a.jsx)(n.p,{children:"or"}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"kubectl apply -f https://kube-vip.io/manifests/rbac.yaml\n"})}),(0,a.jsxs)(n.ol,{start:"2",children:["\n",(0,a.jsx)(n.li,{children:"Deploy the kube-vip daemonset:"}),"\n"]}),(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["Update these values before applying:","\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"vip_interface: the network interface name on each control-plane host (e.g. ens160, eth0)."}),"\n",(0,a.jsx)(n.li,{children:"address: the VIP (example: 10.10.10.100)."}),"\n",(0,a.jsx)(n.li,{children:"node affinity: ensure it matches your control-plane node labels (node-role.kubernetes.io/control-plane vs master)."}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["The list of environment variables is available in the ",(0,a.jsx)(n.a,{href:"https://kube-vip.io/docs/installation/flags/#environment-variables",children:"documentation"}),"."]}),"\n"]}),(0,a.jsxs)(n.p,{children:["Apply the following manifest using the ",(0,a.jsx)(n.code,{children:"kubectl apply -f"})," command."]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yaml",children:'apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n labels:\n app.kubernetes.io/name: kube-vip-ds\n app.kubernetes.io/version: v1.0.4\n name: kube-vip-ds\n namespace: kube-system\nspec:\n selector:\n matchLabels:\n app.kubernetes.io/name: kube-vip-ds\n template:\n metadata:\n labels:\n app.kubernetes.io/name: kube-vip-ds\n app.kubernetes.io/version: v1.0.4\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: node-role.kubernetes.io/master\n operator: Exists\n - matchExpressions:\n - key: node-role.kubernetes.io/control-plane\n operator: Exists\n containers:\n - args:\n - manager\n env:\n - name: vip_arp\n value: "true"\n - name: port\n value: "6443"\n - name: vip_nodename\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: vip_interface\n value: ens160 # <- CHANGE to your host interface or omit\n - name: vip_subnet\n value: "32"\n - name: cp_enable\n value: "true"\n - name: cp_namespace\n value: kube-system\n - name: vip_ddns\n value: "false"\n - name: vip_leaderelection\n value: "true"\n - name: vip_leaseduration\n value: "5"\n - name: vip_renewdeadline\n value: "3"\n - name: vip_retryperiod\n value: "1"\n - name: address\n value: 10.10.10.100 # <- CHANGE to your VIP\n image: ghcr.io/kube-vip/kube-vip:v1.0.4\n imagePullPolicy: Always\n name: kube-vip\n resources: {}\n securityContext:\n capabilities:\n add:\n - NET_ADMIN\n - NET_RAW\n - SYS_TIME\n hostNetwork: true\n serviceAccountName: kube-vip\n tolerations:\n - effect: NoSchedule\n operator: Exists\n - effect: NoExecute\n operator: Exists\n updateStrategy: {}\n'})}),(0,a.jsxs)(n.ol,{start:"3",children:["\n",(0,a.jsx)(n.li,{children:"Verify kube-vip and VIP announcement"}),"\n"]}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"# check pods\nkubectl -n kube-system get pods -l app.kubernetes.io/name=kube-vip-ds\n\n# on a control-plane host, confirm the VIP is in the ARP/neighbor table\nip neigh show | grep 10.10.10.100\n"})}),(0,a.jsxs)(n.ol,{start:"4",children:["\n",(0,a.jsx)(n.li,{children:"TLS certificate note"}),"\n"]}),(0,a.jsx)(n.p,{children:"If K3s was installed before the VIP was added to the API server certificate SANs, kubelets and API clients will not trust the server certificate for the VIP. To include the VIP in server certificates:"}),(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"# Stop K3s service\nsystemctl stop k3s\n\n# Rotate server certificates to include the configured tls-san/VIP\nk3s certificate rotate\n\n# Start K3s service\nsystemctl start k3s\n"})}),(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["After rotation, verify API access using the VIP: kubectl --server=",(0,a.jsx)(n.a,{href:"https://10.10.10.100:6443",children:"https://10.10.10.100:6443"})," get nodes"]}),"\n"]})]})]})]})}function h(e={}){const{wrapper:n}={...(0,t.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(d,{...e})}):d(e)}function p(e,n){throw new Error("Expected "+(n?"component":"object")+" `"+e+"` to be defined: you likely forgot to import, pass, or provide it.")}},28453:(e,n,s)=>{s.d(n,{R:()=>l,x:()=>i});var r=s(96540);const a={},t=r.createContext(a);function l(e){const n=r.useContext(t);return r.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function i(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:l(e.components),r.createElement(t.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.