https://gawsoft.com/component---src-pages-blog-mdx-frontmatter-slu…e-litefs-consul-mdx-f9327789573ee3f6b7d7.js

1"use strict";(self.webpackChunkfrontend=self.webpackChunkfrontend||[]).push([[492],{3056:function(e,t,n){var a=n(3460),l=n(6952);function r(e){const t=Object.assign({p:"p",hr:"hr",h2:"h2",a:"a",ol:"ol",li:"li",strong:"strong",ul:"ul",code:"code",blockquote:"blockquote",pre:"pre"},(0,a.M)(),e.components);return l.createElement(l.Fragment,null,l.createElement(t.p,null,"Headscale, the open-source, self-hosted alternative to Tailscale, is designed to be easy to use, light weight, and it utilizes SQLite primarily as its database.\nHeadscale does not support high availability (HA) natively, nor does it support active-active server configuration, however there are options for resilience in production."),"\n",l.createElement(t.p,null,"In this article, I will explain, as much as I can, my reasoning for the particular architecture I chose which was SQLite, Consul, and automatic failover and the reasons\nI did not choose alternatives such as PostgreSQL."),"\n",l.createElement(t.hr),"\n",l.createElement(t.h2,null,"Why Not PostgreSQL?"),"\n",l.createElement(t.p,null,"Headscale is built with SQLite as its storage and has built-in assumptions that you have a single-writer database.\nWhile PostgreSQL may seem like a natural fit for an HA setup, it creates some serious weight on the project. Here are some main considerations:"),"\n",l.createElement(t.p,null,"Referring to post ",l.createElement(t.a,{href:"https://github.com/juanfont/headscale/issues/2695#issuecomment-3106637622"},"Feature multiple replicas of headscale instances #2695")," of the main contributor of Headscale ",l.createElement(t.a,{href:"https://github.com/kradalby"},"kradalby"),":"),"\n",l.createElement(t.ol,null,"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Complexity Overhead"),": Using PostgreSQL requires more overhead for setup, replication, leader-abdication, and potential migration complexities that add even more risk of bugs and operational overhead."),"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Headscale Mindset"),": The Headscale project is intended as lightweight and easy. As the originator of Headscale has stated, adding multi-server sync brings in a lot more complexity and potential for split-brain scenarios. For small to medium size deployments, the added complexity and risk rarely justify the benefits."),"\n"),"\n",l.createElement(t.p,null,"SQLite is really the best choice for Headscale because it is easier to use, easier to move around, and stays true to the design goals of Headscale."),"\n",l.createElement(t.hr),"\n",l.createElement(t.h2,null,"The Challenge"),"\n",l.createElement(t.p,null,"Even with SQLite, proper HA is difficult to achieve. If the primary Headscale server goes down, we need to be able to keep the database consistent and have a new leader take over."),"\n",l.createElement(t.p,null,"SQLite cannot be safely used concurrently, across multiple nodes, in a real master-replicas scheme, we had to consider different options."),"\n",l.createElement(t.p,null,"It's also worth mentioning that options like ",l.createElement(t.strong,null,l.createElement(t.a,{href:"https://github.com/rqlite/rqlite"},"Rqlite"))," or other SQLite adapters would also work, although this would require to add a pull request with new strategy for that database connection to the project and add overall complexity."),"\n",l.createElement(t.hr),"\n",l.createElement(t.h2,null,"Lessons from the Headscale Developer"),"\n",l.createElement(t.p,null,"The Headscale developer has communicated that multi-server setups are explicitly not supported for reasons of simplicity and reliability. In summary:"),"\n",l.createElement(t.ul,null,"\n",l.createElement(t.li,null,"Full server synchronization across multiple hosts adds complexities to replication and runtime configuration, making debugging problematic."),"\n",l.createElement(t.li,null,"Headscale's design assumes network redundancy at the client level, so momentary outages are generally acceptable."),"\n",l.createElement(t.li,null,"Simple backup-and-restore is generally a better option than trying to implement HA in a distributed way, so my solution aligns with this philosophy."),"\n"),"\n",l.createElement(t.p,null,"I have implemented practical resilience without entirely sacrificing design intent using Consul and replicating SQLite."),"\n",l.createElement(t.hr),"\n",l.createElement(t.h2,null,"My Solution: Headscale + Consul + Auto Failover"),"\n",l.createElement(t.p,null,"To achieve resilience while respecting Headscale’s SQLite design, I implemented a small project with the following architecture:"),"\n",l.createElement(t.p,null,l.createElement(t.strong,null,"Code is available here: ",l.createElement(t.a,{href:"https://github.com/gawsoftpl/headscale-litefs-consul"},"https://github.com/gawsoftpl/headscale-litefs-consul"))),"\n",l.createElement(t.ol,null,"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Primary Headscale Server"),": Handles all client requests and writes to SQLite."),"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Replica Headscale Server"),": Stays in standby mode, ready to take over if the primary fails."),"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Consul for Coordination and Failover"),": Consul monitors the health of the primary server and automatically elects a new leader if the primary fails."),"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"SQLite Replication"),": The SQLite database is continuously replicated by ",l.createElement(t.strong,null,"LiteFS")," to a backup server. Upon failover, the backup server is promoted to primary, and Headscale starts using the replicated database."),"\n"),"\n",l.createElement(t.p,null,"This approach provides:"),"\n",l.createElement(t.ul,null,"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Automatic Failover"),": If the master fails, a new server is automatically selected and brought online with the latest database state."),"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Minimal Complexity"),": No need for multiple active Headscale nodes or complex PostgreSQL replication."),"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Fast Recovery"),": Even if downtime occurs, a new server can be auto restored in 15 seconds using the replicated SQLite database."),"\n"),"\n",l.createElement(t.hr),"\n",l.createElement(t.h2,null,"Disadvantages"),"\n",l.createElement(t.ol,null,"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Consistency")," - It should be remembered that this solution does not ensure consistency.\nWhen primary headscale goes down during write operation, some data may be lost, because ",l.createElement(t.code,null,"LiteFS"),"\nfails during sending the data to the replica node."),"\n",l.createElement(t.li,null,l.createElement(t.strong,null,"Replication data encryption")," - LiteFS replication is not encrypted, because this project was design for ",l.createElement(t.strong,null,"fly.io")," cloud and that platform encrypt traffic by ",l.createElement(t.code,null,"wireguard"),".\nTherefore my solution in production require to add additional network layer to encrypt traffic between LiteFS nodes."),"\n"),"\n",l.createElement(t.hr),"\n",l.createElement(t.h2,null,"What happens if the coordination server is down?"),"\n",l.createElement(t.p,null,"According to post ",l.createElement(t.a,{href:"https://tailscale.com/kb/1091/what-happens-if-the-coordination-server-is-down"},"What happens if the coordination server is down?"),".\nIf the coordination server (headscale) goes down your Tailscale network will mostly ",l.createElement(t.strong,null,"continue to function"),"."),"\n",l.createElement(t.blockquote,null,"\n",l.createElement(t.p,null,"On the other hand, without the coordination server in place:"),"\n",l.createElement(t.ul,null,"\n",l.createElement(t.li,null,"New users and devices cannot be added to the network."),"\n",l.createElement(t.li,null,"Keys cannot be refreshed and exchanged, meaning that existing devices will gradually lose access to each other."),"\n",l.createElement(t.li,null,"Firewall rules cannot be updated."),"\n",l.createElement(t.li,null,"Existing users cannot have their keys revoked.\nsource: ",l.createElement(t.a,{href:"https://tailscale.com/kb/1091/what-happens-if-the-coordination-server-is-down"},"tailscale.com/kb/1091/what-happens-if-the-coordination-server-is-down")),"\n"),"\n"),"\n",l.createElement(t.p,null,"That why 30 seconds of downtime is not a big deal in this scenario. I think master node with 1 replica and auto failover is good enough for most use cases."),"\n",l.createElement(t.hr),"\n",l.createElement(t.h2,null,"A different way to provide high availability is to use Keepalived."),"\n",l.createElement(t.p,null,l.createElement(t.a,{href:"https://github.com/acassen/keepalived"},"Keepalived")," can manage virtual IP addresses and can perform automatic failover between Headscale nodes.\nIn this situation, there could be a number of Headscale servers, but only one would be able to server client requests at a time (the one with the virtual IP)."),"\n",l.createElement(t.p,null,"When the primary node dies, Keepalived can move the virtual IP to a backup node, which would be able to serve clients and still appear to be the primary node."),"\n",l.createElement(t.p,null,"While this makes doing failover at the network level easier, it does not address SQLite's replication problem, for that scenario we have to use ",l.createElement(t.code,null,"LiteFS")," or ",l.createElement(t.a,{href:"https://litestream.io"},"LiteStream"),"."),"\n",l.createElement(t.hr),"\n",l.createElement(t.h2,null,"Demo"),"\n",l.createElement(t.pre,null,l.createElement(t.code,{className:"language-sh"},"# Clone repository\ngit clone https://github.com/gawsoftpl/headscale-litefs-consul\ncd headscale-litefs-consul\n")),"\n",l.createElement(t.pre,null,l.createElement(t.code,{className:"language-sh"}

1,'# Execute docker compose stack\ndocker-compose up -d \necho "Sleeping for 20s for first election and execute servers..."\nsleep 20\n')),"\n",l.createElement(t.p,null,"Those commands emulate failover of primary and auto switch to another node by consule leader election:"),"\n",l.createElement(t.pre,null,l.createElement(t.code,{className:"language-sh"},'echo "Run status report"\nscripts/status.sh\n\n# Get actual leader\nleader_name=`scripts/leader.sh`\necho "Actual leader: $leader_name"\n\n# For demo we create users in headscale\ndocker-compose exec $leader_name headscale users create test test2\n\n# List headscale users, bacause list users saved in primary node\ndocker-compose exec $leader_name headscale users list\n\necho "Emulate failover of primary"\nscripts/failover-test.sh\n\n# Get new leader\nleader_name=`scripts/leader.sh`\necho "Actual leader: $leader_name"\n\necho "Now headscale should run in second node"\ndocker-compose exec $leader_name headscale users list\n')),"\n",l.createElement(t.hr),"\n",l.createElement(t.h2,null,"Conclusion"),"\n",l.createElement(t.p,null,"Headscale may not natively support high availability, but reliable failover can still be accomplished without leaving SQLite behind.\nBy utilizing ",l.createElement(t.code,null,"Consul")," for leader election and database replication by ",l.createElement(t.code,null,"LiteFS")," we can be confident that the Headscale service will still work in the event our server goes down."),"\n",l.createElement(t.p,null,"Additionally, this architecture minimally complicates deployment by avoiding PostgreSQL, is true to the intended use of SQLite in headscale, and provides a\nrespectively robust, and maintainable deployment approach for production builds."))}t.c=function(e){void 0===e&&(e={});const{wrapper:t}=Object.assign({},(0,a.M)(),e.components);return t?l.createElement(t,e,l.createElement(r,e)):r(e)}},9372:function(e,t,n){n.r(t),n.d(t,{Head:function(){return s},default:function(){return c}});var a=n(3056),l=n(6952),r=n(688),o=n(1940);const i=e=>{let{data:t,children:n}=e;return l.createElement(o.K,{post:t.mdx,pageText:n,otherPosts:t.allMdx.nodes,backTo:{name:"Back to blog list",url:"/blog"}})},s=e=>{var t,n,a;let{data:o}=e;return l.createElement(r.c,{title:null!==(t=null===(n=o.mdx.frontmatter)||void 0===n?void 0:n.meta_title)&&void 0!==t?t:null===(a=o.mdx.frontmatter)||void 0===a?void 0:a.title,post:o.mdx,url:"/blog/"+o.mdx.frontmatter.slug+"/"})};function c(e){return l.createElement(i,e,l.createElement(a.c,e))}}}]);
2//# sourceMappingURL=component---src-pages-blog-mdx-frontmatter-slug-tsx-content-file-path-src-content-blog-headscale-litefs-consul-mdx-f9327789573ee3f6b7d7.js.map