https://project-hami.netlify.app/assets/js/8251ee4e.429610bf.js

1"use strict";(globalThis.webpackChunk_project_hami_website||=[]).push([[5788],{23459(e,i,n){n.r(i),n.d(i,{assets:()=>l,contentTitle:()=>a,default:()=>h,frontMatter:()=>r,metadata:()=>t,toc:()=>d});const t=JSON.parse('{"id":"installation/gke-installation","title":"Install HAMi on Google Kubernetes Engine","description":"This guide covers the GKE-specific configuration for installing HAMi with NVIDIA GPU Operator on GKE Standard nodes using the UBUNTU_CONTAINERD image. GPU Operator is not supported on GKE Autopilot.","source":"@site/docs/installation/gke-installation.md","sourceDirName":"installation","slug":"/installation/gke-installation","permalink":"/docs/next/installation/gke-installation","draft":false,"unlisted":false,"editUrl":"https://github.com/Project-HAMi/website/edit/master/docs/installation/gke-installation.md","tags":[],"version":"current","lastUpdatedAt":1789462243000,"frontMatter":{"title":"Install HAMi on Google Kubernetes Engine","sidebar_label":"HAMi on GKE"},"sidebar":"docs","previous":{"title":"HAMi on K3s","permalink":"/docs/next/installation/k3s-installation"},"next":{"title":"HAMi on TKE","permalink":"/docs/next/installation/tke-installation"}}');var o=n(74848),s=n(28453);const r={title:"Install HAMi on Google Kubernetes Engine",sidebar_label:"HAMi on GKE"},a=void 0,l={},d=[{value:"Choose who manages the driver",id:"choose-who-manages-the-driver",level:2},{value:"Let GPU Operator install the driver",id:"let-gpu-operator-install-the-driver",level:3},{value:"Use a GKE-installed driver",id:"use-a-gke-installed-driver",level:3},{value:"Set the containerd configuration source",id:"set-the-containerd-configuration-source",level:2},{value:"Allow critical-priority Pods",id:"allow-critical-priority-pods",level:2},{value:"Troubleshooting",id:"troubleshooting",level:2},{value:"GPU devices missing with CDI disabled",id:"missing-gpu-devices-with-cdi-disabled",level:3},{value:"Next steps",id:"next-steps",level:2}];function c(e){const i={a:"a",admonition:"admonition",code:"code",h2:"h2",h3:"h3",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",...(0,s.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsxs)(i.p,{children:["This guide covers the GKE-specific configuration for installing HAMi with NVIDIA GPU Operator on ",(0,o.jsx)(i.strong,{children:"GKE Standard"})," nodes using the ",(0,o.jsx)(i.code,{children:"UBUNTU_CONTAINERD"})," image. GPU Operator is not supported on GKE Autopilot."]}),"\n",(0,o.jsxs)(i.p,{children:["Start with ",(0,o.jsx)(i.a,{href:"/docs/next/installation/prerequisites",children:"Prerequisites"})," for the shared node requirements. Apply the settings below before following the ",(0,o.jsx)(i.a,{href:"https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/26.3/getting-started.html",children:"GPU Operator installation guide"})," and ",(0,o.jsx)(i.a,{href:"/docs/next/installation/online-installation",children:"Online Installation from Helm"}),". For CDI configuration, see ",(0,o.jsx)(i.a,{href:"/docs/next/installation/configure-cdi",children:"NVIDIA CDI support"}),"."]}),"\n",(0,o.jsx)(i.h2,{id:"choose-who-manages-the-driver",children:"Choose who manages the driver"}),"\n",(0,o.jsxs)(i.p,{children:["GKE can install NVIDIA drivers on GPU nodes. Decide whether GKE or GPU Operator will manage the driver before preparing the node pool. See ",(0,o.jsx)(i.a,{href:"https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/26.3/google-gke.html",children:"NVIDIA's GKE guide"})," for the two installation methods."]}),"\n",(0,o.jsxs)(i.p,{children:["For either method, set ",(0,o.jsx)(i.code,{children:"gke-no-default-nvidia-gpu-device-plugin=true"})," in the GPU node pool's ",(0,o.jsx)(i.code,{children:"--node-labels"})," to disable GKE's NVIDIA Device Plugin. Preserve other required labels, including HAMi's ",(0,o.jsx)(i.code,{children:"gpu=on"})," label. Also set GPU Operator's ",(0,o.jsx)(i.code,{children:"devicePlugin.enabled=false"})," so that HAMi handles GPU registration."]}),"\n",(0,o.jsx)(i.h3,{id:"let-gpu-operator-install-the-driver",children:"Let GPU Operator install the driver"}

1),"\n",(0,o.jsxs)(i.p,{children:["Set ",(0,o.jsx)(i.code,{children:"gpu-driver-version=disabled"})," in the node pool's ",(0,o.jsx)(i.code,{children:"--accelerator"})," option to disable GKE's automatic driver installation. Enable the Operator's driver and Toolkit components with ",(0,o.jsx)(i.code,{children:"driver.enabled=true"})," and ",(0,o.jsx)(i.code,{children:"toolkit.enabled=true"}),"."]}),"\n",(0,o.jsx)(i.h3,{id:"use-a-gke-installed-driver",children:"Use a GKE-installed driver"}),"\n",(0,o.jsxs)(i.p,{children:["Ensure the driver is installed before installing GPU Operator. For new or replacement nodes, follow the ",(0,o.jsx)(i.a,{href:"https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/26.3/google-gke.html#using-the-google-driver-installer",children:"Google driver installer workflow"}),"."]}),"\n",(0,o.jsx)(i.admonition,{title:"Automatic driver installation",type:"note",children:(0,o.jsxs)(i.p,{children:["On GKE ",(0,o.jsx)(i.code,{children:"1.34.10-gke.1328000"})," Ubuntu nodes, GKE's NVIDIA Device Plugin Pod also contains the driver installer. The ",(0,o.jsx)(i.code,{children:"gke-no-default-nvidia-gpu-device-plugin=true"})," label prevents the entire Pod from running, including driver installation. Do not rely on ",(0,o.jsx)(i.code,{children:"gpu-driver-version=default"})," alone to install drivers on new nodes with this label. Prepare the driver installer separately using the workflow above."]})}),"\n",(0,o.jsx)(i.p,{children:"Add the following settings to the GPU Operator values file:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-yaml",children:"driver:\n  enabled: false\nhostPaths:\n  driverInstallDir: /home/kubernetes/bin/nvidia\ntoolkit:\n  installDir: /home/kubernetes/bin/nvidia\n"})}),"\n",(0,o.jsxs)(i.p,{children:["GKE uses ",(0,o.jsx)(i.code,{children:"/home/kubernetes/bin/nvidia"})," for driver files and Toolkit installation. When merging these values, preserve the other Toolkit settings, including ",(0,o.jsx)(i.code,{children:"toolkit.env"})," from the next section."]}),"\n",(0,o.jsxs)(i.p,{children:["Set HAMi's ",(0,o.jsx)(i.code,{children:"devicePlugin.nvidiaDriverRoot=/"})," to use the host filesystem, where GPU device nodes are under ",(0,o.jsx)(i.code,{children:"/dev"}),". For CDI, use ",(0,o.jsx)(i.code,{children:"/home/kubernetes/bin/nvidia/toolkit/nvidia-ctk"})," as ",(0,o.jsx)(i.code,{children:"devicePlugin.nvidiaHookPath"})," in the ",(0,o.jsx)(i.a,{href:"/docs/next/installation/configure-cdi#configure-the-helm-chart",children:"CDI configuration"}),"."]}),"\n",(0,o.jsxs)(i.p,{children:["With CDI enabled, the Google driver installer layout works without an additional ",(0,o.jsx)(i.code,{children:"nvidia-smi"})," symlink. If containers cannot access GPU devices after disabling Operator CDI, see ",(0,o.jsx)(i.a,{href:"#missing-gpu-devices-with-cdi-disabled",children:"GPU devices missing with CDI disabled"}),"."]}),"\n",(0,o.jsx)(i.h2,{id:"set-the-containerd-configuration-source",children:"Set the containerd configuration source"}),"\n",(0,o.jsxs)(i.p,{children:["On ",(0,o.jsx)(i.strong,{children:"GKE 1.33 or later"}),", add this setting to the GPU Operator values file:"]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-yaml",children:"toolkit:\n  env:\n    - name: RUNTIME_CONFIG_SOURCE\n      value: file\n"})}),"\n",(0,o.jsxs)(i.p,{children:["This makes Toolkit read the containerd configuration file directly. It avoids a ",(0,o.jsx)(i.a,{href:"https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/26.3/google-gke.html#prerequisites",children:"known GKE issue"})," that can cause Toolkit to misconfigure containerd and prevent Operator containers from starting. Apply this 

1setting whether CDI is enabled or disabled, and preserve any other required ",(0,o.jsx)(i.code,{children:"toolkit.env"})," entries."]}),"\n",(0,o.jsx)(i.h2,{id:"allow-critical-priority-pods",children:"Allow critical-priority Pods"}),"\n",(0,o.jsxs)(i.p,{children:["If you install the HAMi chart in ",(0,o.jsx)(i.code,{children:"kube-system"}),", no additional ResourceQuota is needed for HAMi. GKE requires this quota only for Pods with ",(0,o.jsx)(i.code,{children:"system-node-critical"})," or ",(0,o.jsx)(i.code,{children:"system-cluster-critical"})," priority in other namespaces."]}),"\n",(0,o.jsxs)(i.p,{children:["GPU Operator still needs the quota if installed outside ",(0,o.jsx)(i.code,{children:"kube-system"}),". For example, when installing it in ",(0,o.jsx)(i.code,{children:"gpu-operator"}),", first create the namespace if it does not already exist:"]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-bash",children:"kubectl create namespace gpu-operator\n"})}),"\n",(0,o.jsxs)(i.p,{children:["Save the following as ",(0,o.jsx)(i.code,{children:"critical-pods-quota.yaml"}),":"]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-yaml",children:'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: critical-pods\nspec:\n  hard:\n    pods: "100"\n  scopeSelector:\n    matchExpressions:\n      - operator: In\n        scopeName: PriorityClass\n        values:\n          - system-node-critical\n          - system-cluster-critical\n'})}),"\n",(0,o.jsx)(i.p,{children:"Apply the quota to the Operator namespace:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-bash",children:"kubectl apply -n gpu-operator -f critical-pods-quota.yaml\n"})}),"\n",(0,o.jsxs)(i.p,{children:["If installing HAMi outside ",(0,o.jsx)(i.code,{children:"kube-system"}),", create its namespace and apply the same quota there before installation."]}),"\n",(0,o.jsx)(i.h2,{id:"troubleshooting",children:"Troubleshooting"}),"\n",(0,o.jsx)(i.h3,{id:"missing-gpu-devices-with-cdi-disabled",children:"GPU devices missing with CDI disabled"}),"\n",(0,o.jsxs)(i.p,{children:["When using Google-installed drivers on GKE Ubuntu nodes, disabling GPU Operator CDI can leave containers without GPU devices and cause HAMi monitor to report ",(0,o.jsx)(i.code,{children:"NVML: Driver Not Loaded"}),". Apply the following driver-path patch to resolve this issue."]}),"\n",(0,o.jsxs)(i.p,{children:["Apply the ",(0,o.jsx)(i.a,{target:"_blank","data-noBrokenLinkCheck":!0,href:n(55513).A+"",children:"driver-path DaemonSet"}),". It creates the ",(0,o.jsx)(i.code,{children:"/usr/bin/nvidia-smi"})," symlink on Ubuntu nodes labeled ",(0,o.jsx)(i.code,{children:"gpu=on"}),", so GPU Operator can recognize the host driver:"]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-bash",children:"kubectl apply -f https://project-hami.io/examples/gke-nvidia-driver-path.yaml\nkubectl rollout status -n kube-system daemonset/gke-nvidia-driver-path\n"})}),"\n",(0,o.jsxs)(i.p,{children:["If GPU Operator and HAMi are already installed, restart their components to apply the fix. Replace ",(0,o.jsx)(i.code,{children:"gpu-operator"})," and ",(0,o.jsx)(i.code,{children:"hami-system"})," in the commands below with the namespaces where GPU Operator and HAMi are installed, respectively:"]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-bash",children:"kubectl rollout restart -n gpu-operator daemonset/nvidia-container-toolkit-daemonset\nkubectl rollout status -n gpu-operator daemonset/nvidia-container-toolkit-daemonset\nkubectl rollout restart -n gpu-operator daemonset/nvidia-operator-validator\nkubectl rollout restart -n hami-system daemonset/hami-device-plugin\nkubectl rollout status -n hami-system daemonset/hami-device-plugin\n"})}),"\n",(0,o.jsx)(i.p,{children:"Create a new GPU workload to verify device access and confirm that HAMi monitor starts normally. Keep the DaemonSet to apply the same fix to replacement nodes."}),"\n",(0,o.jsx)(i.h2,{id:"next-steps",children:"Next steps"}),"\n",(0,o.jsxs)(i.ol,{children:["\n",(0,o.jsxs)(i.li,{children:["Follow the ",(0,o.jsx)(i.a,{href:"https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/26.3/google-gke.html",children:"GPU Operator installation guide for GKE"}),", using the values prepared above, and confirm that its components are ready."]}),"\n",(0,o.jsxs)(i.li,{children:["If using CDI, complete the ",(0,o.jsx)(i.a,{href:"/docs/next/installation/configure-cdi",children:"NVIDIA CDI configuration"})," with the driver root and Toolkit path for your nodes."]}),"\n",(0,o.jsxs)(i.li,{children:["Follow ",(0,o.jsx)(i.a,{href:"/docs/next/installation/online-installation",children:"Online Installation from Helm"})," to install HAMi with your values file and verify the installation."]}),"\n"]})]})}function h(e={}){const{wrapper:i}={...(0,s.R)(),...e.components};return i?(0,o.jsx)(i,{...e,children:(0,o.jsx)(c,{...e})}):c(e)}},55513(e,i,n){const t=n.p+"assets/files/gke-nvidia-driver-path-da9d8adcac59c71b999013b2a261fe48.yaml";n.d(i,["A",0,t])},28453(e,i,n){n.d(i,{R:()=>r,x:()=>a});var t=n(96540);const o={},s=t.createContext(o);function r(e){const i=t.useContext(s);return t.useMemo((function(){return"function"==typeof e?e(i):{...i,...e}}),[i,e])}function a(e){let i;return i=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:r(e.components),t.createElement(s.Provider,{value:i},e.children)}}}]);