https://23blocks.com/developers/assets/js/97f6a42b.7d908861.js

1"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3784],{535:e=>{e.exports=JSON.parse('{"permalink":"/developers/blog/graduating-to-your-own-infrastructure","source":"@site/blog/2026-07-24-graduating-to-your-own-infrastructure/index.md","title":"Graduating to Your Own Infrastructure: An Anonymized Case Study","description":"How one customer moved 11 backend blocks across 4 consumer surfaces off shared 23blocks and onto their own cloud, in a single reversible maintenance window \u2014 sequences, JWT issuers, and all.","date":"2026-07-24T00:00:00.000Z","tags":[{"inline":false,"label":"Engineering","permalink":"/developers/blog/tags/engineering","description":"Engineering deep-dives and technical guides"},{"inline":false,"label":"Infrastructure","permalink":"/developers/blog/tags/infrastructure","description":"Infrastructure, deployment, and graduation stories"},{"inline":false,"label":"Open Source","permalink":"/developers/blog/tags/open-source","description":"Open-sourcing 23blocks blocks and related announcements"}],"readingTime":6.65,"hasTruncateMarker":true,"authors":[{"name":"Juan Pel\xe1ez","title":"Founder and CTO of 23blocks","url":"https://juanpelaez.com","page":{"permalink":"/developers/blog/authors/all-juan-pelaez-articles"},"socials":{"x":"https://x.com/jkpelaez","linkedin":"https://www.linkedin.com/in/juancarlospelaez/","github":"https://github.com/juan-23blocks"},"imageURL":"https://github.com/slorber.png","key":"jkpelaez"}],"frontMatter":{"slug":"graduating-to-your-own-infrastructure","title":"Graduating to Your Own Infrastructure: An Anonymized Case Study","authors":["jkpelaez"],"tags":["engineering","infrastructure","open-source"],"description":"How one customer moved 11 backend blocks across 4 consumer surfaces off shared 23blocks and onto their own cloud, in a single reversible maintenance window \u2014 sequences, JWT issuers, and all.","image":"/developers/img/blog/graduating-to-your-own-infrastructure.png"},"unlisted":false,"prevItem":{"title":"We\'re Open-Sourcing All of 23blocks","permalink":"/developers/blog/open-sourcing-23blocks"},"nextItem":{"title":"23blocks Platform Architecture: Edge Infrastructure & Environment Routing","permalink":"/developers/blog/23blocks-platform-architecture"}}')},1184:(e,n,t)=>{t.d(n,{R:()=>o,x:()=>a});var s=t(4041);const i={},r=s.createContext(i);function o(e){const n=s.useContext(r);return s.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:o(e.components),s.createElement(r.Provider,{value:n},e.children)}},8894:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>l,contentTitle:()=>a,default:()=>h,frontMatter:()=>o,metadata:()=>s,toc:()=>d});var s=t(535),i=t(1085),r=t(1184);const o={slug:"graduating-to-your-own-infrastructure",title:"Graduating to Your Own Infrastructure: An Anonymized Case Study",authors:["jkpelaez"],tags:["engineering","infrastructure","open-source"],description:"How one customer moved 11 backend blocks across 4 consumer surfaces off shared 23blocks and onto their own cloud, in a single reversible maintenance window \u2014 sequences, JWT issuers, and all.",image:"/developers/img/blog/graduating-to-your-own-infrastructure.png"},a=void 0,l={authorsImageUrls:[void 0]},d=[{value:"The setup",id:"the-setup",level:2},{value:"Six phases, four of them invisible to users",id:"six-phases-four-of-them-invisible-to-users",level:2},{value:"The cutover window",id:"the-cutover-window",level:2},{value:"Five lessons that are now baked into the tooling",id:"five-lessons-that-are-now-baked-into-the-tooling",level:2},{value:"1. A data re-sync silently reverts infra-config",id:"1-a-data-re-sync-silently-reverts-infra-config",level:3},{value:"2. Env-var-driven frontends don&#39;t flip when you deploy code",id:"2-env-var-driven-frontends-dont-flip-when-you-deploy-code",level:3},{value:"3. Postgres sequences don&#39;t travel with <code>pg_dump</code>",id:"3-postgres-sequences-dont-travel-with-pg_dump",level:3},{value:"4. JWT issuer identity must match the validating host, exactly",id:"4-jwt-issuer-identity-must-match-the-validating-host-exactly",level:3},{value:"5. Config drift across blocks is real and it will bite you at the worst time",id:"5-config-drift-across-blocks-is-real-and-it-will-bite-you-at-the-worst-time",level:3},{value:"Why we&#39;re writing this down publicly",id:"why-were-writing-this-down-publicly",level:2}];function c(e){const n={a:"a",code:"code",em:"em",h2:"h2",h3:"h3",li:"li",ol:"ol",p:"p",strong:"strong",...(0,r.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(n.p,{children:"A customer of ours \u2014 a casting/talent platform, details anonymized \u2014 recently moved 11 backend blocks and 4 consumer surfaces off our shared platform and onto their own cloud account. Zero code rewrite. One reversible maintenance window. This is the technical writeup, because the lessons are more useful shared than kept as internal tribal knowledge."}),"\n",(0,i.jsxs)(n.p,{children:["Full reference: ",(0,i.jsx)(n.a,{href:"/developers/docs/platform/in-a-box",children:"23blocks in a Box \u2014 the engineering guide"}),"."]}),"\n",(0,i.jsx)(n.h2,{id:"the-setup",children:"The setup"}),"\n",(0,i.jsx)(n.p,{children:"The customer was running 11 blocks (auth, files, forms, CRM, products, search, and others) on our shared, multi-tenant platform, consumed by 4 separate surfaces \u2014 a public web app, an admin console, a mobile client, and an internal ops tool. They wanted dedicated infrastru

1cture in their own AWS account: their own compliance boundary, their own scaling, their own on-call, no shared-tenant blast radius."}),"\n",(0,i.jsxs)(n.p,{children:["The property that made this tractable at all: ",(0,i.jsx)(n.strong,{children:"every block is fully configured by environment variables."})," No hardcoded hostnames, no baked-in secrets, the same container image runs on shared or dedicated infrastructure unchanged. Graduating a block isn't a code change \u2014 it's a question of where the container runs and what URLs the consuming apps point at."]}),"\n",(0,i.jsx)(n.h2,{id:"six-phases-four-of-them-invisible-to-users",children:"Six phases, four of them invisible to users"}),"\n",(0,i.jsxs)(n.p,{children:[(0,i.jsx)(n.strong,{children:"1. Fork."})," Each block forked into the customer's own org. No CI secrets or shared-platform config travel with it \u2014 just the code, which they now own outright and can ",(0,i.jsx)(n.code,{children:"git pull upstream"})," on their own schedule."]}),"\n",(0,i.jsxs)(n.p,{children:[(0,i.jsx)(n.strong,{children:"2. Build."})," The fork's own pipeline builds images, pushes to the customer's registry. Their build, their artifacts."]}),"\n",(0,i.jsxs)(n.p,{children:[(0,i.jsx)(n.strong,{children:"3. Provision."})," Dedicated single-tenant footprint stood up in their AWS account \u2014 same architecture as shared 23blocks, sized for one tenant: load balancer, container services, a Postgres instance with one database per block, Redis, S3-equivalent storage, TLS."]}),"\n",(0,i.jsxs)(n.p,{children:[(0,i.jsx)(n.strong,{children:"4. Migrate data."})," Per-block tenant schemas copied whole; control rows filtered to the tenant; sequences advanced to ",(0,i.jsx)(n.code,{children:"MAX(id)"}),"; row parity verified table by table."]}),"\n",(0,i.jsx)(n.p,{children:"Phases 1\u20134 happened over roughly two weeks with zero user-facing impact. Phases 5 and 6 \u2014 cutting over consumers and verifying \u2014 happened inside a single maintenance window."}),"\n",(0,i.jsx)(n.h2,{id:"the-cutover-window",children:"The cutover window"}),"\n",(0,i.jsx)(n.p,{children:"The flip itself is designed to be boring, because boring is reversible and exciting is not."}),"\n",(0,i.jsxs)(n.ol,{children:["\n",(0,i.jsxs)(n.li,{children:[(0,i.jsx)(n.strong,{children:"Freeze."})," All 4 surfaces stop writing \u2014 SPAs show a maintenance page, backends go process-down. One very-low-write internal tool took an accepted no-freeze."]}),"\n",(0,i.jsxs)(n.li,{children:[(0,i.jsx)(n.strong,{children:"Final delta re-sync."})," Bring the dedicated database to an exact, current copy of the shared tenant. Reset sequences again. Verify parity again."]}),"\n",(0,i.jsxs)(n.li,{children:[(0,i.jsx)(n.strong,{children:"URL-only flip."})," Every consumer's only change is a base URL: ",(0,i.jsx)(n.code,{children:"*.api.us.23blocks.com"})," \u2192 ",(0,i.jsx)(n.code,{children:"*.api.us.<customer-domain>"}),". Same API key, valid on both sides. No redeploy of application logic, no new secrets to distribute."]}),"\n",(0,i.jsxs)(n.li,{children:[(0,i.jsx)(n.strong,{children:"New major version per app."})," Each of the 4 surfaces cut a new major version whose ",(0,i.jsx)(n.em,{children:"only"}),' diff is the URLs. New major = own infrastructure; previous major = shared. The version number alone now tells you where an app points, and rollback is "redeploy the previous major" \u2014 no rebuild required.']}),"\n",(0,i.jsxs)(n.li,{children:[(0,i.jsx)(n.strong,{children:"Verify behind the maintenance page."})," Auth sign-in and token validation, the full read path, one real write \u2014 checked before anyone is let back in."]}),"\n",(0,i.jsxs)(n.li,{children:[(0,i.jsx)(n.strong,{children:"Confirm zero shared traffic."}),' Post-flip, the shared load balancer showed zero requests carrying that tenant\'s API key, across all 11 blocks. That\'s the actual definition of "done" \u2014 not "we flipped the switch," but "we can see nobody\'s using the old path anymore."']}),"\n"]}),"\n",(0,i.jsx)(n.p,{children:"Total user-facing disruption: one maintenance window. Rollback stayed armed the entire time because the shared-cloud data was never mutated during the process \u2014 flipping URLs back is always clean."}),"\n",(0,i.jsx)(n.h2,{id:"five-lessons-that-are-now-baked-into-the-tooling",children:"Five lessons that are now baked into the tooling"}),"\n",(0,i.jsx)(n.p,{children:"These weren't known in advance. They were found the hard 

1way, on real graduations, and are now checks the migration tooling runs automatically."}),"\n",(0,i.jsx)(n.h3,{id:"1-a-data-re-sync-silently-reverts-infra-config",children:"1. A data re-sync silently reverts infra-config"}),"\n",(0,i.jsx)(n.p,{children:"The tenant's control row carries infrastructure settings alongside business data \u2014 the auth JWT issuer URL, the API base URL, webhook URLs \u2014 and on shared, all of them point at shared-cloud hosts. A naive re-sync overwrites the customer's own values with the shared ones."}),"\n",(0,i.jsxs)(n.p,{children:["This is exactly what happened: after a delta re-sync, the issuer URL silently reverted, the customer's auth service started minting tokens with the ",(0,i.jsx)(n.em,{children:"shared"})," issuer, and every login started failing validation on the new infrastructure. Nothing was down; auth just rejected its own freshly-issued tokens."]}),"\n",(0,i.jsxs)(n.p,{children:[(0,i.jsx)(n.strong,{children:"Fix:"})," the migration now captures the target's infra-config columns before every re-sync and restores them immediately after. Data moves; infra config doesn't get overwritten by it."]}),"\n",(0,i.jsx)(n.h3,{id:"2-env-var-driven-frontends-dont-flip-when-you-deploy-code",children:"2. Env-var-driven frontends don't flip when you deploy code"}),"\n",(0,i.jsxs)(n.p,{children:["One of the four surfaces was a Next.js site with ",(0,i.jsx)(n.code,{children:"NEXT_PUBLIC_*"})," API URLs baked in ",(0,i.jsx)(n.strong,{children:"at build time"})," from the hosting console's environment variables \u2014 not read from the repo at runtime. Shipping new application code did nothing to the URLs; the console env vars had to be updated ",(0,i.jsx)(n.em,{children:"and the site rebuilt"})," before it would talk to the new infrastructure."]}),"\n",(0,i.jsxs)(n.p,{children:["The team's other surfaces, Angular apps carrying config in ",(0,i.jsx)(n.code,{children:"environment.prod.ts"}),", didn't have this trap \u2014 their config is compiled into the code artifact itself, so a new build picks it up automatically. The lesson isn't \"one pattern is better,\" it's: ",(0,i.jsx)(n.strong,{children:"know which config model each of your surfaces uses before the cutover window"}),", because the remediation step is different for each, and you don't want to discover that live."]}),"\n",(0,i.jsxs)(n.h3,{id:"3-postgres-sequences-dont-travel-with-pg_dump",children:["3. Postgres sequences don't travel with ",(0,i.jsx)(n.code,{children:"pg_dump"})]}),"\n",(0,i.jsxs)(n.p,{children:[(0,i.jsx)(n.code,{children:"pg_dump"})," / ",(0,i.jsx)(n.code,{children:"COPY"}),' moves rows. It does not move sequence state. The first insert against a freshly copied table collides on a primary key that already "exists" as far as the sequence\'s internal counter is concerned, because that counter reset to whatever it was in the dump, not to ',(0,i.jsx)(n.code,{children:"MAX(id)"}),"."]}),"\n",(0,i.jsxs)(n.p,{children:["This is easy to miss in a dry run with low write volume and then bite hard in production the moment real traffic resumes writes. ",(0,i.jsxs)(n.strong,{children:["Every serial sequence gets explicitly advanced to ",(0,i.jsx)(n.code,{children:"MAX(id)"})," after every copy \u2014 dry run or real cutover, no exceptions."]})]}),"\n",(0,i.jsx)(n.h3,{id:"4-jwt-issuer-identity-must-match-the-validating-host-exactly",children:"4. JWT issuer identity must match the validating host, exactly"}),"\n",(0,i.jsx)(n.p,{children:"If a token's issuer claim doesn't match the host validating it, the login bounces \u2014 not because credentials are wrong, but because the token was minted for a different identity than the one checking it. This surfaced in the same incident as lesson #1, but it's worth stating as its own rule because it applies any time issuer and validator can get out of lockstep \u2014 for instance, if backend services can't all flip in the same instant."}),"\n",(0,i.jsxs)(n.p,{children:[(0,i.jsx)(n.strong,{children:"Mitigation for staged rollouts:"})," a dual-trust issuer window \u2014 accepting both the old and new issuer for a bounded period \u2014 smooths a cutover where not every service flips atomically."]}),"\n",(0,i.jsx)(n.h3,{id:"5-config-drift-across-blocks-is-real-and-it-will-bite-you-at-the-worst-time",children:"5. Config drift across blocks is real and it will bite you at the worst time"}),"\n",(0,i.jsx)(n.p,{children:"Values that are supposed to be identical across every block's database \u2014 the tenant's API key, shared webhook URLs \u2014 tend to drift over the life of a tenant, usually from manual hotfixes applied to one block and never backported to the others. Verify these are consistent across every block before you plan a cutover, not during it. Discovering a mismatched API key mid-maintenance-window turns a 30-minute flip into a 3-hour one."}),"\n",(0,i.jsx)(n.h2,{id:"why-were-writing-this-down-publicly",children:"Why we're writing this down publicly"}),"\n",(0,i.jsx)(n.p,{children:"None of this is proprietary insight worth hoarding \u2014 it's the kind of thing every team doing a data-plane migration eventually rediscovers, usually at 2am. We'd rather you read it here first. It's also, not coincidentally, the same reason we're open-sourcing the blocks themselves: the code and the operational knowledge around it are both more valuable shared than protected, and we'd rather compete on doing this well than on being the only ones who know how."}),"\n",(0,i.jsxs)(n.p,{children:["If you're evaluating a similar move, the full technical reference \u2014 the complete environment-variable contract, health-check endpoints, and the six-phase process in more detail \u2014 is at ",(0,i.jsx)(n.a,{href:"/developers/docs/platform/in-a-box",children:"23blocks in a Box"}),"."]})]})}function h(e={}){const{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,i.jsx)(n,{...e,children:(0,i.jsx)(c,{...e})}):c(e)}}}]);