https://docs.k3s.io/assets/js/a65db7ae.82e38c5e.js

1"use strict";(self.webpackChunkk_3_s_docs=self.webpackChunkk_3_s_docs||[]).push([[1510],{49766:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>o,default:()=>h,frontMatter:()=>a,metadata:()=>i,toc:()=>l});var i=n(21795),s=n(74848),r=n(28453);const a={title:"K3s initialization deep dive",description:"Explain k3s initialization steps",authors:"manuelbuil",hide_table_of_contents:!0},o=void 0,c={authorsImageUrls:[void 0]},l=[{value:"The Embedded Powerhouse \u2699\ufe0f\u26a1",id:"the-embedded-powerhouse-\ufe0f",level:2},{value:"The Boot Sequence, Step-by-Step \ud83d\udc63",id:"the-boot-sequence-step-by-step-",level:2},{value:"Conclusion \ud83c\udfc1",id:"conclusion-",level:2}];function d(e){const t={a:"a",code:"code",h2:"h2",p:"p",pre:"pre",...(0,r.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(t.p,{children:"K3s is a lightweight Kubernetes distribution which excels in its deployment speed and minimal resource footprint. In fact, a lot of our users love K3s because it offers an unparalleled initialization speed."}),"\n",(0,s.jsx)(t.p,{children:"This blog post delves into the heart of K3s's efficie

1ncy: its initialization process. We'll embark on a journey through the steps that enable K3s to materialize a fully functional Kubernetes cluster so quickly. By examining K3s\u2019s own logs, we'll unravel the meaning behind each step, providing you with a practical understanding of how K3s achieves its remarkable speed. This exploration not only illuminates the inner workings of K3s but also equips you with the knowledge to troubleshoot your deployments."}),"\n",(0,s.jsx)(t.h2,{id:"the-embedded-powerhouse-\ufe0f",children:"The Embedded Powerhouse \u2699\ufe0f\u26a1"}),"\n",(0,s.jsxs)(t.p,{children:["K3s leverages ",(0,s.jsx)(t.code,{children:"go-bindata"})," to embed essential Linux userspace binaries and manifests directly into its executable. This eliminates external dependencies and streamlines the deployment process. Within the K3s binary, you'll find core components like ",(0,s.jsx)(t.code,{children:"runc"})," and ",(0,s.jsx)(t.code,{children:"containerd"}),", along with the k3s-root tarball (e.g. k3s-root-amd64.tar). This tarball contains all the userspace binaries necessary for K3s to function, reducing the reliance on the host OS. If you would like all the K3s embedded binaries to take preference over the host OS binaries, you should use the ",(0,s.jsx)(t.code,{children:"--prefer-bundled-bin"})," flag."]}),"\n",(0,s.jsxs)(t.p,{children:["The embedded binaries are always deployed in the same directory: ",(0,s.jsx)(t.code,{children:"/var/lib/rancher/k3s/data"}),". If you inspect this folder, you will notice it contains at least three subdirectories: ",(0,s.jsx)(t.code,{children:"cni"}),", ",(0,s.jsx)(t.code,{children:"current"})," and a long string of characters (or SHA). That long string of characters is generated 

1when building K3s and it is the result of a ",(0,s.jsx)(t.code,{children:"sha256sum"})," operation made on the tarball with the embedded binaries. As these change in each release, you will see a different string of characters for each release. In fact, after an upgrade, there will be two directories with a long string of characters as their name."]}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.code,{children:"current"})," is just a symlink to the SHA directory and ",(0,s.jsx)(t.code,{children:"cni"})," includes different cni plugins that are also symlinks to the cni binary in the SHA directory. This is because we are building all cni plugins in just one binary using multi-exec tooling. This is again a way to be more efficient and less resource consuming. If your current K3s deployment underwent an upgrade process, you will see one extra directory called ",(0,s.jsx)(t.code,{children:"previous"}),", which is another symlink to the previous SHA directory. For clarification this example:"]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"$> ls -ahltr /var/lib/rancher/k3s/data/\ntotal 24K\n-rw------- 1 root root    0 Mar 19 06:22 .lock\ndrwxr-xr-x 4 root root 4.0K Mar 19 06:22 ..\ndrwxr-xr-x 4 root root 4.0K Mar 19 06:28 82142f5157c67effc219aeefe0bc03e0460fc62b9fbae9e901270c86b5635d53\nlrwxrwxrwx 1 root root   90 Mar 19 06:28 previous -> /var/lib/rancher/k3s/data/82142f5157c67effc219aeefe0bc03e0460fc62b9fbae9e901270c86b5635d53\ndrwxr-xr-x 4 root root 4.0K Mar 19 06:30 b13851fe661ab93938fc9a881cdce529da8c6b9b310b2440ef01a860f8b9c3a9\nlrwxrwxrwx 1 root root   90 Mar 19 06:30 current -> /var/lib/rancher/k3s/data/b13851fe661ab93938fc9a881cdce529da8c6b9b310b2440ef01a860f8b9c3a9\ndrwxr-xr-x 2 root root 4.0K Mar 19 06:30 cni\ndrwxr-xr-x 4 root root 4.0K Mar 19 06:40 .\n"})}),"\n",(0,s.jsxs)(t.p,{children:["Additionally, K3s includes embedded Helm charts and manifests for deploying critical services such as CoreDNS, Traefik, and local storage. These embedded charts, formatted as yaml files, can be found in the control plane nodes in the directory: ",(0,s.jsx)(t.code,{children:"/var/lib/rancher/k3s/server/manifests"}),"."]}),"\n",(0,s.jsx)(t.p,{children:"For 67MB our K3s binary includes a lot of stuff!"}),"\n",(0,s.jsx)(t.h2,{id:"the-boot-sequence-step-by-step-",children:"The Boot Sequence, Step-by-Step \ud83d\udc63"}),"\n",(0,s.jsxs)(t.p,{children:["Now that we established how K3s is carrying its embedded tools, we can explore the boot up sequence. Let us look at the typical logs you can find in ",(0,s.jsx)(t.code,{children:"journalctl"})," when deploying a control-plane or K3s server instance. It all starts with:"]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"Starting Lightweight Kubernetes...\n"})}),"\n",(0,s.jsx)(t.p,{children:"And then:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"/usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service\n"})}),"\n",(0,s.jsxs)(t.p,{children:["This part is checking for a network manager utility which must be disabled as described in the ",(0,s.jsx)(t.a,{href:"https://docs.k3s.io/installation/requirements?_highlight=nm&_highlight=cloud&_highlight=setup.service&os=rhel#operating-systems",children:"docs"}),". It configures some parts of the network stack, specifically the routing tables, which conflict with Kubernetes networking, and that is why we verify if it was correctly disabled."]}),"\n",(0,s.jsx)(t.p,{children:"The next message should look familiar"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"Acquiring lock file /var/lib/rancher/k3s/data/.lock\nPreparing data dir /var/lib/rancher/k3s/data/f8e9b5e7d85085972f4a9ddfd539d4dcf887be2e380a55f415c93cac5516dad5\n"})}),"\n",(0,s.jsxs)(t.p,{children:["When this message is shown, the directory where K3s deploys the embedded binaries has already been created. At this point, K3s will extract the binaries. We use the lock to avoid concurrent modifications, preventing K3s embedded commands like ",(0,s.jsx)(t.code,{children:"kubectl"})," or ",(0,s.jsx)(t.code,{children:"ctr"})," from executing and disturbing the K3s initialization."]}),"\n",(0,s.jsxs)(t.p,{children:["The next block of logs point at the K3s version and the datastore. In this case, I am using the default datastore which means kine with sqlite. For more information on the different datastores available check this ",(0,s.jsx)(t.a,{href:"https://docs.k3s.io/datastore",children:"link"})]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"Starting k3s v1.32.2+k3s1\nConfiguring sqlite3 database connection pooling: maxIdleConns=2, maxOpenConns=0, connMaxLifetime=0s\nConfiguring database table schema and indexes, this may take a moment...\nDatabase tables and indexes are up to date\nKine available at unix://kine.s

1ock\n"})}),"\n",(0,s.jsx)(t.p,{children:"Once the datastore is available k3s locks the bootstrap key. This step is useful for HA mode and this this key is just a placeholder so that other control-plane nodes do not start generating new CA certs. As we are not using HA mode in this example, this is not relevant."}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"Bootstrap key locked for initial create\n"})}),"\n",(0,s.jsx)(t.p,{children:"K3s then generates all the TLS certificates required for the internal communications:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"generated self-signed CA certificate\ncertificate CN=system:admin,O=system:masters signed by CN=k3s-client-ca@1742309831\ncertificate CN=system:k3s-supervisor,O=system:masters signed by CN=k3s-client-ca@1742309831\ncertificate CN=system:kube-controller-manager signed by CN=k3s-client-ca@1742309831\ncertificate CN=system:kube-scheduler signed by CN=k3s-client-ca@1742309831\ncertificate CN=system:apiserver,O=system:masters signed by CN=k3s-client-ca@1742309831\ncertificate CN=k3s-cloud-controller-manager signed by CN=k3s-client-ca@1742309831\ngenerated self-signed CA certificate CN=k3s-server-ca@1742309831\ncertificate CN=kube-apiserver signed by CN=k3s-server-ca@1742309831\ngenerated self-signed CA certificate CN=k3s-request-header-ca@1742309831\ncertificate CN=system:auth-proxy signed by CN=k3s-request-header-ca@1742309831\ngenerated self-signed CA certificate CN=etcd-server-ca@1742309831\ncertificate CN=etcd-client signed by CN=etcd-server-ca@1742309831\ngenerated self-signed CA certificate CN=etcd-peer-ca@1742309831\ncertificate CN=etcd-peer signed by CN=etcd-peer-ca@1742309831\ncertificate CN=etcd-server signed by CN=etcd-server-ca@1742309831\ncertificate CN=k3s,O=k3s signed by CN=k3s-server-ca@1742309831\n"})}),"\n",(0,s.jsx)(t.p,{children:"And then saves the bootstrap data in the bootstrap key. Again, this is not relevant for this example:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"Saving cluster bootstrap data to datastore\n"})}),"\n",(0,s.jsx)(t.p,{children:"After that, K3s starts the different Kubernetes components. These components are all run within the k3s process as goroutines, which are lightweight, concurrent functions in Go, allowing for efficient resource usage. This is another design decision taken to reduce boot time and reduce resource consumption."}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"Running kube-apiserver\nRunning kube-scheduler\nRunning kube-controller-manager\n"})}),"\n",(0,s.jsxs)(t.p,{children:["Manifests for packaged components are extracted to ",(0,s.jsx)(t.code,{children:"/var/lib/rancher/k3s/server/manifests/"}),". When all the different Kubernetes components are running and K3s initialization is ready, the deploy controller begins watching this directory and applies all the manifests. This is how components like CoreDNS or Traefik eventually get installed."]}),"\n",(0,s.jsx)(t.p,{children:"And that\u2019s it, in a short period of time, you end up with a fully deployed and running Kubernetes distribution. \ud83c\udf89"}),"\n",(0,s.jsx)(t.h2,{id:"conclusion-",children:"Conclusion \ud83c\udfc1"}),"\n",(0,s.jsx)(t.p,{children:"This exploration has hopefully demystified some of the initial steps that enable K3s to materialize a fully functional Kubernetes cluster. By examining the logs, we've shed some light on the meaning behind each step, providing you with a deeper understanding of how K3s deploys in such a fast manner. We hope you find this knowledge useful to troubleshoot or at least to understand a bit deeper how K3s works."})]})}function h(e={}){const{wrapper:t}={...(0,r.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},28453:(e,t,n)=>{n.d(t,{R:()=>a,x:()=>o});var i=n(96540);const s={},r=i.createContext(s);function a(e){const t=i.useContext(r);return i.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function o(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:a(e.components),i.createElement(r.Provider,{value:t},e.children)}},21795:e=>{e.exports=JSON.parse('{"permalink":"/blog/2025/03/25/K3s-initialization","source":"@site/blog/2025-03-25-K3s-initialization.md","title":"K3s initialization deep dive","description":"Explain k3s initialization steps","date":"2025-03-25T00:00:00.000Z","tags":[],"hasTruncateMarker":true,"authors":[{"title":"K3s maintainer","url":"https://github.com/manuelbuil","name":"Manuel Buil","imageURL":"https://github.com/manuelbuil.png","key":"manuelbuil","page":null}],"frontMatter":{"title":"K3s initialization deep dive","description":"Explain k3s initialization steps","authors":"manuelbuil","hide_table_of_contents":true},"unlisted":false,"prevItem":{"title":"Kubernetes v1.34 is out!","permalink":"/blog/2025/08/30/K3s-1.34-release"},"nextItem":{"title":"The Basic HA Cluster","permalink":"/blog/2025/03/10/simple-ha"}}')}}]);