https://harvesterhci.io/assets/js/56ab5bb4.ed808209.js

1"use strict";(self.webpackChunkharvesterhci_io=self.webpackChunkharvesterhci_io||[]).push([[5962],{3905:function(e,t,r){r.d(t,{Zo:function(){return d},kt:function(){return u}});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?o(Object(r),!0).forEach((function(t){a(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):o(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function s(e,t){if(null==e)return{};var r,n,a=function(e,t){if(null==e)return{};var r,n,a={},o=Object.keys(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)r=o[n],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var l=n.createContext({}),c=function(e){var t=n.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},d=function(e){var t=c(e.components);return n.createElement(l.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},h=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,l=e.parentName,d=s(e,["components","mdxType","originalType","parentName"]),h=c(r),u=a,k=h["".concat(l,".").concat(u)]||h[u]||p[u]||o;return r?n.createElement(k,i(i({ref:t},d),{},{components:r})):n.createElement(k,i({ref:t},d))}));function u(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=h;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var c=2;c<o;c++)i[c]=r[c];return n.createElement.apply(null,i)}return n.createElement.apply(null,r)}h.displayName="MDXCreateElement"},3613:function(e,t,r){r.r(t),r.d(t,{assets:function(){return d},contentTitle:function(){return l},default:function(){return u},frontMatter:function(){return s},metadata:function(){return c},toc:function(){return p}});var n=r(7462),a=r(3366),o=(r(7294),r(3905)),i=["components"],s={title:"Best Practices for Harvester Security",description:"A set of best practices for Harvester security.",slug:"harvester_security_best_practices",authors:[{name:"Jian Wang",title:"Staff Software Engineer",url:"https://github.com/w13915984028",image_url:"https://github.com/w13915984028.png"}],tags:["harvester","security","credential"],hide_table_of_contents:!1},l=void 0,c={permalink:"/kb/harvester_security_best_practices",editUrl:"https://github.com/harvester/harvesterhci.io/edit/main/kb/2024-05-31/harvester_security_best_practice.md",source:"@site/kb/2024-05-31/harvester_security_best_practice.md",title:"Best Practices for Harvester Security",description:"A set of best practices for Harvester security.",date:"2024-05-31T00:00:00.000Z",formattedDate:"May 31, 2024",tags:[{label:"harvester",permalink:"/kb/tags/harvester"},{label:"security",permalink:"/kb/tags/security"},{label:"credential",permalink:"/kb/tags/credential"}],readingTime:4.75,truncated:!1,authors:[{name:"Jian Wang",title:"Staff Software Engineer",url:"https://github.com/w13915984028",image_url:"https://github.com/w13915984028.png",imageURL:"https://github.com/w13915984028.png"}],frontMatter:{title:"Best Practices for Harvester Security",description:"A set of best practices for Harvester security.",slug:"harvester_security_best_practices",authors:[{name:"Jian Wang",title:"Staff Software Engineer",url:"https://github.com/w13915984028",image_url:"https://github.com/w13915984028.png",imageURL:"https://github.com/w13915984028.png"}],tags:["harvester","security","credential"],hide_table_of_contents:!1},prevItem:{title:"Shutdown and Restart a Harvester Cluster",permalink:"/kb/shutdown_and_restart_a_harvester_cluster"},nextItem:{title:"Renew Harvester Cloud Credentials",permalink:"/kb/renew_harvester_cloud_credentials"}},d={authorsImageUrls:[void 0]},p=[{value:"User-Provided Credentials on Harvester",id:"user-prov

1ided-credentials-on-harvester",level:2},{value:"Cluster Token",id:"cluster-token",level:3},{value:"Cluster Token on Nodes Joining an Existing Cluster",id:"cluster-token-on-nodes-joining-an-existing-cluster",level:4},{value:"Cluster Token (RKE2 Token Rotation)",id:"cluster-token-rke2-token-rotation",level:4},{value:"Password of the Default User <code>rancher</code>",id:"password-of-the-default-user-rancher",level:3},{value:"SSH keys",id:"ssh-keys",level:3},{value:"HTTP Proxy",id:"http-proxy",level:3},{value:"Other Credentials and Settings",id:"other-credentials-and-settings",level:2},{value:"<code>auto-rotate-rke2-certs</code>",id:"auto-rotate-rke2-certs",level:3},{value:"Harvester Cloud Credentials",id:"harvester-cloud-credentials",level:3},{value:"<code>additional-ca</code>",id:"additional-ca",level:3},{value:"<code>ssl-certificates</code>",id:"ssl-certificates",level:3},{value:"<code>ssl-parameters</code>",id:"ssl-parameters",level:3},{value:"<code>containerd-registry</code>",id:"containerd-registry",level:3}],h={toc:p};function u(e){var t=e.components,r=(0,a.Z)(e,i);return(0,o.kt)("wrapper",(0,n.Z)({},h,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h2",{id:"user-provided-credentials-on-harvester"},"User-Provided Credentials on Harvester"),(0,o.kt)("p",null,"When ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.2/install/index#installation-steps"},"installing a Harvester cluster"),", you are asked to provide the following credential related information:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("p",{parentName:"li"},"Cluster token of the first node that is added to the cluster. Other nodes must use this token to join the cluster.")),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("p",{parentName:"li"},"Password for the default Linux user ",(0,o.kt)("inlineCode",{parentName:"p"},"rancher")," on each node.")),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("p",{parentName:"li"},"SSH keys on each node (optional).")),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("p",{parentName:"li"},"HTTP proxy on each node (optional)."))),(0,o.kt)("p",null,"You may plan to change them from time to time, the following paragraphs describe the detailed steps."),(0,o.kt)("h3",{id:"cluster-token"},"Cluster Token"),(0,o.kt)("h4",{id:"cluster-token-on-nodes-joining-an-existing-cluster"},"Cluster Token on Nodes Joining an Existing Cluster"),(0,o.kt)("p",null,"When a node is unable to join a cluster because of a cluster token error, perform the recommended ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.2/troubleshooting/index/#modifying-cluster-token-on-agent-nodes"},"troubleshooting steps"),"."),(0,o.kt)("h4",{id:"cluster-token-rke2-token-rotation"},"Cluster Token (RKE2 Token Rotation)"),(0,o.kt)("p",null,"Harvester does not allow you to change the cluster token even if RKE2 is a core component of Harvester."),(0,o.kt)("p",null,"The ",(0,o.kt)("a",{parentName:"p",href:"https://docs.rke2.io/security/token#server-token-rotation"},"RKE2 documentation")," states that the November 2023 releases of RKE2 (v1.28.3+rke2r2, v1.27.7+rke2r2, v1.26.10+rke2r2, and v1.25.15+rke2r2) allow you to rotate the cluster token using the command ",(0,o.kt)("inlineCode",{parentName:"p"},"rke2 token rotate --token original --new-token new"),". "),(0,o.kt)("p",null,"During testing, the command was run on the first node of a cluster running ",(0,o.kt)("strong",{parentName:"p"},"Harvester v1.3.0 with RKE2 v1.27.10+rke2r1"),"."),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},"Rotate the token on initial node.")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"/opt/rke2/bin $ ./rke2 token rotate --token rancher --new-token rancher1\n\nWARNING: Recommended to keep a record of the old token. If restoring from a snapshot, you must use the token associated with that snapshot.\nWARN[0000] Cluster CA certificate is not trusted by the host CA bundle, but the token does not include a CA hash. Use the full token from the server's node-token file to enable Cluster CA validation. \nToken rotated, restart rke2 nodes with new token\n")),(0,o.kt)("ol",{start:2},(0,o.kt)("li",{parentName:"ol"},"When the first cluster node was rebooted, RKE2 service was unable to start.")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},'RKE2 log:\n\n...\nMay 29 15:45:11 harv41 rke2[3293]: time="2024-05-29T15:45:11Z" level=info msg="etcd temporary data store connection OK"\nMay 29 15:45:11 harv41 rke2[3293]: time="2024-05-29T15:45:11Z" level=info msg="Reconciling bootstrap data between datastore and disk"\nMay 29 15:45:11 harv41 rke2[3293]: time="2024-05-29T15:45:11Z" level=fatal msg="Failed to reconcile with temporary etcd: bootstrap data already found and encrypted with different token"\nMay 29 15:45:11 harv41 systemd[1]: rke2-server.service: Main process exited, code=exited, status=1/FAILURE\n...\n')),(0,o.kt)("p",null,"This known issue was logged on Github issue ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/rancher/rke2/issues/6250"},"rke2 token rotate does not work as expected (v1.27.10+rke2r1)"),"."),(0,o.kt)("p",null,":::Warning"),

1(0,o.kt)("p",null,"Do not attempt to rotate the RKE2 token on your cluster before Harvester announces official support for this feature (even if the embedded RKE2 binary has the ",(0,o.kt)("inlineCode",{parentName:"p"},"token rotate")," option)."),(0,o.kt)("p",null,":::"),(0,o.kt)("h3",{id:"password-of-the-default-user-rancher"},"Password of the Default User ",(0,o.kt)("inlineCode",{parentName:"h3"},"rancher")),(0,o.kt)("p",null,"This process is node-specific. You must change the ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.2/install/update-harvester-configuration/#password-of-user-rancher"},"password of the default user")," on each node even if the same password is used on all Harvester nodes."),(0,o.kt)("h3",{id:"ssh-keys"},"SSH keys"),(0,o.kt)("p",null,"You must log into a Harvester node using the default user account ",(0,o.kt)("inlineCode",{parentName:"p"},"rancher")," to change the ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.2/install/update-harvester-configuration#ssh-keys-of-user-rancher"},"SSH keys"),"."),(0,o.kt)("h3",{id:"http-proxy"},"HTTP Proxy"),(0,o.kt)("p",null,"After a Harvester cluster is installed, you can use the Harvester UI to change the ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.2/advanced/index#http-proxy"},"HTTP proxy"),"."),(0,o.kt)("p",null,"Alternatively, you can use ",(0,o.kt)("inlineCode",{parentName:"p"},"kubectl")," or the rest API against the URI ",(0,o.kt)("inlineCode",{parentName:"p"},"/harvesterhci.io.setting/http-proxy"),"."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},'$ kubectl get settings.harvesterhci.io http-proxy -oyaml\n\napiVersion: harvesterhci.io/v1beta1\ndefault: \'{}\'\nkind: Setting\nmetadata:\n  creationTimestamp: "2024-05-13T20:44:20Z"\n  generation: 1\n  name: http-proxy\n  resourceVersion: "5914"\n  uid: 282506bb-f1dd-4247-bf0e-93640698c1f5\nstatus: {}\n')),(0,o.kt)("p",null,"Harvester has a webhook that checks this setting to ensure it meets all conditions, e.g. the internal IPs and CIDRs are specified in the ",(0,o.kt)("inlineCode",{parentName:"p"},"noProxy")," field."),(0,o.kt)("div",{className:"admonition admonition-note alert alert--secondary"},(0,o.kt)("div",{parentName:"div",className:"admonition-heading"},(0,o.kt)("h5",{parentName:"div"},(0,o.kt)("span",{parentName:"h5",className:"admonition-icon"},(0,o.kt)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},(0,o.kt)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),(0,o.kt)("div",{parentName:"div",className:"admonition-content"},(0,o.kt)("p",{parentName:"div"},"Avoid changing the HTTP proxy from files in the host ",(0,o.kt)("inlineCode",{parentName:"p"},"/oem")," path for the following reasons:"),(0,o.kt)("ul",{parentName:"div"},(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("p",{parentName:"li"},"You must manually change the HTTP proxy on each node.")),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("p",{parentName:"li"},"Contents of local files are not automatically populated to new nodes.")),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("p",{parentName:"li"},"Without help from the webhook, some erroneous configurations may not be promptly detected (see ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/harvester/harvester/pull/5824"},"Node IP should be in noProxy"),").")),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("p",{parentName:"li"},"Harvester may change the file naming or content structure in the future."))))),(0,o.kt)("h2",{id:"other-credentials-and-settings"},"Other Credentials and Settings"),(0,o.kt)("h3",{id:"auto-rotate-rke2-certs"},(0,o.kt)("inlineCode",{parentName:"h3"},"auto-rotate-rke2-certs")),(0,o.kt)("p",null,"Harvester is built on top of Kubernetes, RKE2, and Rancher. RKE2 generates a list of ",(0,o.kt)("inlineCode",{parentName:"p"},"*.crt")," and ",(0,o.kt)("inlineCode",{parentName:"p"},"*.key")," files that allow Kubernetes components to function. The ",(0,o.kt)("inlineCode",{parentName:"p"},"*.crt")," file expires after one year by default."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"$ ls /var/lib/rancher/rke2/server/tls/ -alth\n\n...\n-rw-r--r-- 1 root root  570 May 27 08:45 server-ca.nochain.crt\n-rw------- 1 root root 1.7K May 27 08:45 service.current.key\n-rw-r--r-- 1 root root  574 May 27 08:45 client-ca.nochain.crt\ndrw

1xr-xr-x 2 root root 4.0K May 13 20:45 kube-controller-manager\ndrwxr-xr-x 2 root root 4.0K May 13 20:45 kube-scheduler\ndrwx------ 6 root root 4.0K May 13 20:45 .\ndrwx------ 8 root root 4.0K May 13 20:45 ..\n-rw-r--r-- 1 root root 3.9K May 13 20:40 dynamic-cert.json\ndrwx------ 2 root root 4.0K May 13 20:39 temporary-certs\n-rw------- 1 root root 1.7K May 13 20:39 service.key\n-rw-r--r-- 1 root root 1.2K May 13 20:39 client-auth-proxy.crt\n-rw------- 1 root root  227 May 13 20:39 client-auth-proxy.key\n-rw-r--r-- 1 root root 1.2K May 13 20:39 client-rke2-cloud-controller.crt\n...\n-rw-r--r-- 1 root root 1.2K May 13 20:39 client-admin.crt\n-rw------- 1 root root  227 May 13 20:39 client-admin.key\n...\n\n\n$ openssl x509 -enddate -noout -in /var/lib/rancher/rke2/server/tls/client-admin.crt\n\nnotAfter=May 13 20:39:42 2025 GMT\n")),(0,o.kt)("p",null,"When a cluster has been running for over one year, Kubernetes components may fail to start after upgrades or node rebooting. The ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/harvester/harvester/issues/3863#issuecomment-1539681311"},"workaround")," is to delete the related files and restart the pod."),(0,o.kt)("p",null,"Harvester v1.3.0 added the setting ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.3/advanced/index#auto-rotate-rke2-certs"},(0,o.kt)("inlineCode",{parentName:"a"},"auto-rotate-rke2-certs")),", which allows you to set the Harvester cluster to automatically rotate certificates for RKE2 services. When you enable the setting and specify a certificate validity period, Harvester automatically replaces the certificate before the specified period ends."),(0,o.kt)("div",{className:"admonition admonition-note alert alert--secondary"},(0,o.kt)("div",{parentName:"div",className:"admonition-heading"},(0,o.kt)("h5",{parentName:"div"},(0,o.kt)("span",{parentName:"h5",className:"admonition-icon"},(0,o.kt)("svg",{parentName:"span",xmlns:"http://www.w3.org/2000/svg",width:"14",height:"16",viewBox:"0 0 14 16"},(0,o.kt)("path",{parentName:"svg",fillRule:"evenodd",d:"M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"}))),"note")),(0,o.kt)("div",{parentName:"div",className:"admonition-content"},(0,o.kt)("p",{parentName:"div"},"Enabling this setting on your cluster is highly recommended."))),(0,o.kt)("h3",{id:"harvester-cloud-credentials"},"Harvester Cloud Credentials"),(0,o.kt)("p",null,"See the article ",(0,o.kt)("a",{parentName:"p",href:"https://harvesterhci.io/kb/renew_harvester_cloud_credentials"},"Renew Harvester Cloud Credentials"),"."),(0,o.kt)("h3",{id:"additional-ca"},(0,o.kt)("inlineCode",{parentName:"h3"},"additional-ca")),(0,o.kt)("p",null,"See the ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.2/advanced/index#additional-ca"},"documentation")," for this setting."),(0,o.kt)("h3",{id:"ssl-certificates"},(0,o.kt)("inlineCode",{parentName:"h3"},"ssl-certificates")),(0,o.kt)("p",null,"See the ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.2/advanced/index#ssl-certificates"},"documentation")," for this setting."),(0,o.kt)("h3",{id:"ssl-parameters"},(0,o.kt)("inlineCode",{parentName:"h3"},"ssl-parameters")),(0,o.kt)("p",null,"See the ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.2/advanced/index#ssl-parameters"},"documentation")," for this setting."),(0,o.kt)("h3",{id:"containerd-registry"},(0,o.kt)("inlineCode",{parentName:"h3"},"containerd-registry")),(0,o.kt)("p",null,"See the ",(0,o.kt)("a",{parentName:"p",href:"https://docs.harvesterhci.io/v1.2/advanced/index#containerd-registry"},"documentation")," for this setting."))}u.isMDXComponent=!0}}]);