https://kairos.io/assets/js/24875f22.10343640.js

1"use strict";(self.webpackChunkkairos_docs=self.webpackChunkkairos_docs||[]).push([["28715"],{3747(e,n,t){t.r(n),t.d(n,{metadata:()=>i,default:()=>u,frontMatter:()=>a,contentTitle:()=>o,toc:()=>l,assets:()=>d});var i=JSON.parse('{"id":"examples/kdump","title":"Enabling kdump","description":"This section describe examples on how to enable kdump in Kairos derivatives","source":"@site/versioned_docs/version-v4.1.2/examples/kdump.md","sourceDirName":"examples","slug":"/examples/kdump","permalink":"/docs/v4.1.2/examples/kdump","draft":false,"unlisted":false,"editUrl":"https://github.com/kairos-io/kairos-docs/tree/main/versioned_docs/version-v4.1.2/examples/kdump.md","tags":[],"version":"v4.1.2","lastUpdatedBy":"github-actions[bot]","lastUpdatedAt":1782202763000,"frontMatter":{"title":"Enabling kdump","sidebar_label":"Enabling kdump","description":"This section describe examples on how to enable kdump in Kairos derivatives"},"sidebar":"tutorialSidebar","previous":{"title":"Run stages along with K3s","permalink":"/docs/v4.1.2/examples/k3s-stages"},"next":{"title":"Keylime agent","permalink":"/docs/v4.1.2/examples/keylime"}}'),s=t(74848),r=t(28453);let a={title:"Enabling kdump",sidebar_label:"Enabling kdump",description:"This section describe examples on how to enable kdump in Kairos derivatives"},o="Introduction",d={},l=[{value:"Building the custom derivative",id:"building-the-custom-derivative",level:3},{value:"Build an iso from that artifact",id:"build-an-iso-from-that-artifact",level:3},{value:"Install the iso",id:"install-the-iso",level:3},{value:"Check that kdump is enabled and works",id:"check-that-kdump-is-enabled-and-works",level:3}];function c(e){let n={a:"a",admonition:"admonition",code:"code",h1:"h1",h3:"h3",header:"header",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,r.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.admonition,{title:"Info",type:"info",children:(0,s.jsx)(n.p,{children:"This tutorial is based on Opensuse Leap. Kdump configs vary over distributions and we are not able to test them all but they should be easily adaptable from this tutorial."})}),"\n",(0,s.jsx)(n.header,{children:(0,s.jsx)(n.h1,{id:"introduction",children:"Introduction"})}),"\n",(0,s.jsx)(n.p,{children:"kdump is a feature of the Linux kernel that creates crash dumps in the event of a kernel crash. When triggered, kdump exports a memory image (also known as vmcore) that can be analyzed for the purposes of debugging and determining the cause of a crash."}),"\n",(0,s.jsx)(n.p,{children:"In the event of a kernel crash, kdump preserves system consistency by booting another Linux kernel, which is known as the dump-capture kernel, and using it to export and save a memory dump. As a result, the system boots into a clean and reliable environment instead of relying on an already crashed kernel that may cause various issues, such as causing file system corruption while writing a memory dump file"}),"\n",(0,s.jsxs)(n.p,{children:["This is why we need a clean initramfs that disables most of the modules and mounts the persistent partition into the ",(0,s.jsx)(n.code,{children:"/var/crash"})," route in order to store the dumps over reboots"]}),"\n",(0,s.jsx)(n.h1,{id:"requirements",children:"Requirements"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"A custom image that builds a simple,small initrd with the kdump module and that mounts persistent to store the dump"}),"\n",(0,s.jsx)(n.li,{children:"A service override to skip the kdump service rebuilding the initrd on an immutable system"}),"\n"]}),"\n",(0,s.jsx)(n.h1,{id:"steps",children:"Steps"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Build the custom derivative artifact"}),"\n",(0,s.jsx)(n.li,{children:"Build an iso from that artifact"}),"\n",(0,s.jsx)(n.li,{children:"Install the iso"}),"\n",(0,s.jsx)(n.li,{children:"Check that kdump is enabled and works"}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"building-the-custom-derivative",children:"Building the custom derivative"}),"\n",(0,s.jsxs)(n.p,{children:["We will keep this short as there is more docs about building your own derivatives than what we can go in this tutorial like the ",(0,s.jsx)(n.a,{href:"/docs/advanced/customizing/",children:"Customizing page"})]}),"\n",(0,s.jsx)(n.p,{children:"The main step is to build a clean initrd that has the kdump module and can mount persistent to store the kernel dump."}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-dockerfile",children:'FROM quay.io/kairos/opensuse:leap-15.6-core-amd64-generic-v3.1.2\n\n# Install kdump\nRUN zypper ref && zypper in -y kdump kexec-tools makedumpfile\n# Build initramfs with kdump module on it\nRUN kernel=$(ls /lib/modules | head -n1) && depmod -a "${kernel}" && \\\n            dracut -N -f /var/lib/kdump/initrd "${kernel}" -a kdump \\\n            --omit "plymouth resume usrmount zz-fadumpinit immucore kairos-network kairos-sysext" --compress "xz -0 --check=crc32" \\\n            --mount "/dev/disk/by-label/COS_PE

1RSISTENT /kdump/mnt/var/crash ext4 rw,relatime"\n# On opensuse the path to the kernel is hardcoded so we need to soft link it to our kernel\nRUN ln -s /boot/vmlinuz /var/lib/kdump/kernel\n# Enable services. early service is the one on the initramfs\nRUN systemctl enable kdump-early && systemctl enable kdump\n'})}),"\n",(0,s.jsxs)(n.p,{children:["We are generating a new initrd and storing it on ",(0,s.jsx)(n.code,{children:"/var/lib/kdump/initrd"})," as the kdump service will look into that directory to find the kernel and initrd needed to exec into.\nWe are using the following options to generate a simple, clean initrd:"]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"-a kdump"}),": adds the kdump module explicitly to the initramfs."]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"--omit"}),": omits modules from initrd. This is to have a clean, simple initramfs."]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"--compress"}),": Compresses the initrd to keep it small."]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.code,{children:"--mount"}),": Explicitly mount the PERSISTENT partition into ",(0,s.jsx)(n.code,{children:"/kdump/mnt/var/crash"})," so kdump can store the dump"]}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:"Then we generate a new artifact using that dockerfile:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'$ docker build -t kdump-kairos .\n[+] Building 0.6s (9/9) FINISHED                                                         docker:default\n => [internal] load build definition from Dockerfile                                               0.0s\n => => transferring dockerfile: 853B                                                               0.0s\n => [internal] load metadata for quay.io/kairos/opensuse:leap-15.6-core-amd64-generic-v3.1.2       0.6s\n => [internal] load .dockerignore                                                                  0.0s\n => => transferring context: 2B                                                                    0.0s\n => [1/5] FROM quay.io/kairos/opensuse:leap-15.6-core-amd64-generic-v3.1.2@sha256:a85cf92ea9ed5a0  0.0s\n => CACHED [2/5] RUN zypper ref && zypper in -y kdump kexec-tools makedumpfile                     0.0s\n => CACHED [3/5] RUN kernel=$(ls /lib/modules | head -n1) && depmod -a "${kernel}" &&              0.0s\n => CACHED [4/5] RUN ln -s /boot/vmlinuz /var/lib/kdump/kernel                                     0.0s\n => CACHED [5/5] RUN systemctl enable kdump-early && systemctl enable kdump                        0.0s\n => exporting to image                                                                             0.0s\n => => exporting layers                                                                            0.0s\n => => writing image sha256:a7ed8f51a32648f8e97cae1eb253a309b696dc3243ba366b489a76150721f403       0.0s\n => => naming to docker.io/library/kdump-kairos           \n'})}),"\n",(0,s.jsx)(n.h3,{id:"build-an-iso-from-that-artifact",children:"Build an iso from that artifact"}),"\n",(0,s.jsxs)(n.p,{children:["Again, this tutorial does not cover this part deeply as there are docs providing a deep insight onto this like the ",(0,s.jsx)(n.a,{href:"/docs/reference/auroraboot/",children:"AuroraBoot page"})]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"$ docker run -v \"$PWD\"/build-iso:/tmp/auroraboot -v /var/run/docker.sock:/var/run/docker.sock --rm -ti quay.io/kairos/auroraboot --set container_image=\"oci:kdump-kairos\" --set \"disable_http_server=true\" --set \"disable_netboot=true\" --set \"state_dir=/tmp/auroraboot\"\n2:38PM INF Pulling container image 'kdump-kairos' to '/tmp/auroraboot/temp-rootfs' (local: true)\n2:38PM INF Generating iso 'kairos' from '/tmp/auroraboot/temp-rootfs' to '/tmp/auroraboot/build'\n"})}),"\n",(0,s.jsx)(n.h3,{id:"install-the-iso",children:"Install the iso"}),"\n",(0,s.jsx)(n.p,{children:"Then we burn the resulting ISO to a dvd or usb stick and boot it normally."}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"In order to have kdump working properly, we need to reserve a chunk of memory from the system so it can dump correctly"}),". Several tools exist for this like ",(0,s.jsx)(n.code,{children:"k

1dumptool"})," which will give us some approximated values to reserve if running on the machine. A safe value might be ",(0,s.jsx)(n.code,{children:"512M high"})," and ",(0,s.jsx)(n.code,{children:"72M low"})]}),"\n",(0,s.jsxs)(n.p,{children:["This values need to be passed to the kernel in the cmdline so kdump knows what memory it has to work with. The easiest way is to set the ",(0,s.jsx)(n.code,{children:"install.grub_options.extra_cmdline"})," value in the ",(0,s.jsx)(n.a,{href:"/docs/reference/configuration/",children:"cloud-config"})," during install."]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:'#cloud-config\n\ninstall:\n  auto: true\n  reboot: true\n  grub_options:\n    extra_cmdline: "crashkernel=512M,high crashkernel=72M,low"\n\nstages:\n  initramfs:\n    - name: "Set user and password"\n      users:\n        kairos:\n          passwd: "kairos"\n    - name: "Set hostname"\n      hostname: kairos-{{ trunc 4 .Random }}\n'})}),"\n",(0,s.jsx)(n.h3,{id:"check-that-kdump-is-enabled-and-works",children:"Check that kdump is enabled and works"}),"\n",(0,s.jsxs)(n.p,{children:["Once the system has been installed there is 2 services that can be checked to see if kdump is correctly enabled. ",(0,s.jsx)(n.code,{children:"kdump-early"})," and ",(0,s.jsx)(n.code,{children:"kdump"})," services run in initrafms and userspace respectively and both of them should be in status ",(0,s.jsx)(n.code,{children:"Active"})," after booting:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"$ systemctl status kdump-early\n* kdump-early.service - Load kdump kernel early on startup\n     Loaded: loaded (/usr/l

1ib/systemd/system/kdump-early.service; enabled; preset: disabled)\n     Active: active (exited) since Mon 2024-09-09 14:41:02 UTC; 19min ago\n   Main PID: 1556 (code=exited, status=0/SUCCESS)\n        CPU: 68ms\n\nSep 09 14:41:02 kairos-cfig systemd[1]: Starting Load kdump kernel early on startup...\nSep 09 14:41:02 kairos-cfig systemd[1]: Finished Load kdump kernel early on startup.\n\n$ systemctl status kdump      \n* kdump.service - Load kdump kernel and initrd\n     Loaded: loaded (/usr/lib/systemd/system/kdump.service; enabled; preset: disabled)\n     Active: active (exited) since Mon 2024-09-09 14:41:03 UTC; 20min ago\n   Main PID: 1672 (code=exited, status=0/SUCCESS)\n        CPU: 64ms\n\nSep 09 14:41:03 kairos-cfig systemd[1]: Starting Load kdump kernel and initrd...\nSep 09 14:41:03 kairos-cfig systemd[1]: Finished Load kdump kernel and initrd.\n"})}),"\n",(0,s.jsx)(n.p,{children:"Now we know that our systems is kdump ready and in case of a kernel crash it will dump the crash allowing us to troubleshoot it."}),"\n",(0,s.jsxs)(n.p,{children:["The dumps will be stored in ",(0,s.jsx)(n.code,{children:"/usr/local/DATE"})," and will survive reboots"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"$ ls -ltra /usr/local/2024-09-09-15-03/\ntotal 79488\ndrwxr-xr-x 9 root root     4096 Sep  9 15:03 ..\n-rw-r--r-- 1 root root    74108 Sep  9 15:03 dmesg\ndrwxr-xr-x 2 root root     4096 Sep  9 15:03 .\n-rw-r--r-- 1 root root 81305093 Sep  9 15:03 vmcore\n-rw-r--r-- 1 root root      320 Sep  9 15:03 README.txt\n"})}),"\n",(0,s.jsxs)(n.admonition,{title:"Warning",type:"warning",children:[(0,s.jsxs)(n.p,{children:["You can manually trigger a crash by running ",(0,s.jsx)(n.code,{children:"echo c > /proc/sysrq-trigger"})]}),(0,s.jsx)(n.p,{children:"Note that this will immediately crash your machine, dump the kernel and restart so make sure that everything is ready for the sudden crash."})]})]})}function u(e={}){let{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(c,{...e})}):c(e)}},28453(e,n,t){t.d(n,{R:()=>a,x:()=>o});var i=t(96540);let s={},r=i.createContext(s);function a(e){let n=i.useContext(r);return i.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function o(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:a(e.components),i.createElement(r.Provider,{value:n},e.children)}}}]);