https://limitr.dev/assets/js/3964a926.28c34d56.js

1"use strict";(globalThis.webpackChunklimitr_site||=[]).push([[51],{862(e,t,n){n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>o,default:()=>h,frontMatter:()=>a,metadata:()=>r,toc:()=>l});let r=JSON.parse('{"id":"guides/testingpolicy","title":"Testing Your Policy","description":"Validation as a CI gate, enforcement tests for limits, events, grants, and feature gates, and testing policy changes before shipping.","source":"@site/docs/guides/testingpolicy.mdx","sourceDirName":"guides","slug":"/guides/testingpolicy","permalink":"/guides/testingpolicy","draft":false,"unlisted":false,"tags":[],"version":"current","lastUpdatedAt":1785420178000,"sidebarPosition":4,"frontMatter":{"title":"Testing Your Policy","sidebar_position":4,"description":"Validation as a CI gate, enforcement tests for limits, events, grants, and feature gates, and testing policy changes before shipping."},"sidebar":"guidesSidebar","previous":{"title":"Handling Plan Changes","permalink":"/guides/planchanges"},"next":{"title":"Cloud Quick Start","permalink":"/guides/cloud/quickstart"}}');var i=n(4848),s=n(8453);let a={title:"Testing Your Policy",sidebar_position:4,description:"Validation as a CI gate, enforcement tests for limits, events, grants, and feature gates, and testing policy changes before shipping."},o="Testing Your Policy",c={},l=[{value:"What the built-in validator catches",id:"validator",level:2},{value:"Validation as a CI gate",id:"ci",level:2},{value:"Writing enforcement tests",id:"tests",level:2},{value:"Setup pattern",id:"setup-pattern",level:3},{value:"Testing hard limits",id:"testing-hard-limits",level:3},{value:"Testing soft limits and events",id:"testing-soft-limits-and-events",level:3},{value:"Testing grant coverage",id:"testing-grant-coverage",level:3},{value:"Testing boolean feature gates",id:"testing-boolean-feature-gates",level:3},{value:"Testing customer overrides",id:"testing-customer-overrides",level:3},{value:"Testing policy changes with <code>difference()</code>",id:"difference",level:2},{value:"What to test before shipping a pricing change",id:"shipping",level:2}];function d(e){let t={admonition:"admonition",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",hr:"hr",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,s.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(t.header,{children:(0,i.jsx)(t.h1,{id:"testing-your-policy",children:"Testing Your Policy"})}),"\n",(0,i.jsx)(t.p,{children:"Self-hosted testing."}),"\n",(0,i.jsx)(t.admonition,{title:"Using Limitr Cloud?",type:"tip",children:(0,i.jsxs)(t.p,{children:["Policy validation is built into the Cloud publishing workflow. Cloud maintains separate test and live environments \u2014 make changes in test, verify behavior in your development environment connected to the test policy, publish to live when it's right. You don't need a separate test suite for policy correctness in Cloud. The rest of this guide is for developers using the ",(0,i.jsx)(t.strong,{children:"local open source engine"}),"."]})}),"\n",(0,i.jsx)(t.hr,{}),"\n",(0,i.jsx)(t.h2,{id:"validator",children:"What the built-in validator catches"}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.code,{children:"Limitr.new()"})," validates your policy by default before returning. If the policy is invalid, it throws with the validation error."]}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",children:"try {\r\n  const policy = await Limitr.new(policyYaml, 'yaml');\r\n} catch (err) {\r\n  console.error('Invalid policy:', err.message);\r\n  // e.g. 'A credit named \"sonnet_inpput\" does not exist in this policy'\r\n}\n"})}),"\n",(0,i.jsxs)(t.p,{children:["You can also call ",(0,i.jsx)(t.code,{children:"valid()"})," explicitly on an already-loaded policy:"]}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",children:"const [valid, error] = await policy.valid();\r\nif (!valid) console.error(error);\n"})}),"\n",(0,i.jsx)(t.p,{children:"The validator checks credit references, unit strings, limit and topup values, tier structure, entitlement and topup schema conformance, and trial period configuration."}),"\n",(0,i.jsxs)(t.p,{children:["What the validator does ",(0,i.jsx)(t.strong,{children:"not"})," check:"]}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsx)(t.li,{children:"Whether your credit prices and costs make economic sense"}),"\n",(0,i.jsx)(t.li,{children:"Whether your limits are appropriate for your product"}),"\n",(0,i.jsx)(t.li,{children:"Whether your exchange table chains resolve correctly for your use case"}),"\n",(0,i.jsx)(t.li,{children:"Whether enforcement behaves the way you expect at runtime"}),"\n"]}),"\n",(0,i.jsx)(t.p,{children:"Those require behavioral tests."}),"\n",(0,i.jsx)(t.hr,{}),"\n",(0,i.jsx)(t.h2,{id:"ci",children:"Validation as a CI gate"}),"\n",(0,i.jsx)(t.p,{children:"The simplest thing you can do: load your policy file in CI and let the built-in validator fail the build on an invali

1d policy. No test framework required."}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",metastring:'title="scripts/validate-policy.ts"',children:"import { Limitr } from '@formata/limitr';\r\nimport { readFileSync } from 'fs';\r\n\r\nconst policyPath = process.argv[2] ?? './policy.yaml';\r\n\r\ntry {\r\n  const policy = await Limitr.new(readFileSync(policyPath, 'utf-8'), 'yaml');\r\n  const version = await policy.version();\r\n  console.log(`\u2713 Policy valid (Limitr ${version})`);\r\n  process.exit(0);\r\n} catch (err) {\r\n  console.error(`\u2717 Invalid policy: ${(err as Error).message}`);\r\n  process.exit(1);\r\n}\n"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-json",metastring:'title="package.json"',children:'"scripts": {\r\n  "validate": "bun run scripts/validate-policy.ts policy.yaml"\r\n}\n'})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-yaml",metastring:'title=".github/workflows/ci.yml"',children:"- name: Validate Limitr policy\r\n  run: bun run validate\n"})}),"\n",(0,i.jsx)(t.hr,{}),"\n",(0,i.jsx)(t.h2,{id:"tests",children:"Writing enforcement tests"}),"\n",(0,i.jsxs)(t.p,{children:["Enforcement tests verify that ",(0,i.jsx)(t.code,{children:"allow()"}),", ",(0,i.jsx)(t.code,{children:"increment()"}),", and ",(0,i.jsx)(t.code,{children:"check()"})," behave correctly for the scenarios your product depends on. Use your existing test framework \u2014 Limitr is just TypeScript."]}),"\n",(0,i.jsx)(t.h3,{id:"setup-pattern",children:"Setup pattern"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",children:"import { Limitr } from '@formata/limitr';\r\nimport { readFileSync } from 'fs';\r\n\r\nasync function loadPolicy() {\r\n  return Limitr.new(readFileSync('./policy.yaml', 'utf-8'), 'yaml');\r\n}\n"})}),"\n",(0,i.jsxs)(t.p,{children:["Create a fresh policy instance per test or test file. Limitr is in-process \u2014 there's no server to reset and no shared state. Each ",(0,i.jsx)(t.code,{children:"Limitr.new()"})," call is a clean slate."]}),"\n",(0,i.jsx)(t.h3,{id:"testing-hard-limits",children:"Testing hard limits"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",children:"test('starter hard limit blocks at 500K input tokens', async () => {\r\n  const policy = await loadPolicy();\r\n  await policy.createCustomer('test_user', 'starter');\r\n\r\n  const allowed = await policy.allow('test_user', 'chat_input', 500_000);\r\n  expect(allowed).toBe(true);\r\n\r\n  const blocked = await policy.allow('test_user', 'chat_input', 1);\r\n  expect(blocked).toBe(false);\r\n\r\n  const remaining = await policy.remaining('test_user', 'chat_input');\r\n  expect(remaining).toBe(0);\r\n});\n"})}),"\n",(0,i.jsx)(t.h3,{id:"testing-soft-limits-and-events",children:"Testing soft limits and events"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",children:"test('growth soft limit fires meter-overage event', async () => {\r\n  const policy = await loadPolicy();\r\n  await policy.createCustomer('test_user', 'growth');\r\n\r\n  const events: string[] = [];\r\n  policy.addHandler('test', (key) => events.push(key));\r\n\r\n  await policy.allow('test_user', 'chat_input', 2_000_000); // at limit\r\n  await policy.allow('test_user', 'chat_input', 1);          // over limit\r\n\r\n  expect(events).toContain('meter-overage');\r\n  expect(events).not.toContain('meter-limit');\r\n});\n"})}),"\n",(0,i.jsx)(t.h3,{id:"testing-grant-coverage",children:"Testing grant coverage"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",children:"test('grant covers overage before meter-overage fires', async () => {\r\n  const policy = await loadPolicy();\r\n  await policy.createCustomer('test_user', 'growth');\r\n  await policy.applyCustomerTopup('test_user', 'monthly_credits');\r\n\r\n  const overageEvents: unknown[] = [];\r\n  policy.addHandler('test', (key, value) => {\r\n    if (key === 'meter-overage') overageEvents.push(JSON.parse(value as string));\r\n  });\r\n\r\n  await policy.allow('test_user', 'chat_input', 2_000_000); // at limit\r\n  await policy.allow('test_user', 'chat_input', 100_000);   // into overage \u2014 covered by grant\r\n\r\n  expect(overageEvents).toHaveLength(0); // grant covered it\r\n\r\n  await policy.allow('test_user', 'chat_input', 10_000_000); // exhaust the grant\r\n\r\n  expect(overageEvents.length).toBeGreaterThan(0);\r\n});\n"})}),"\n",(0,i.jsx)(t.h3,{id:"testing-boolean-feature-gates",children:"Testing boolean feature gates"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",children:"test('starter plan does not have advanced analytics', async () => {\r\n  const policy = await loadPolicy();

1\r\n  await policy.createCustomer('test_starter', 'starter');\r\n  await policy.createCustomer('test_growth', 'growth');\r\n\r\n  expect(await policy.check('test_starter', 'advanced_analytics')).toBe(false);\r\n  expect(await policy.check('test_growth', 'advanced_analytics')).toBe(true);\r\n});\n"})}),"\n",(0,i.jsx)(t.h3,{id:"testing-customer-overrides",children:"Testing customer overrides"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",children:"test('customer override replaces plan limit', async () => {\r\n  const policy = await loadPolicy();\r\n  await policy.createCustomer('test_user', 'starter');\r\n\r\n  expect(await policy.limit('test_user', 'chat_input', false)).toBe(500_000);\r\n\r\n  await policy.createCustomerOverride('test_user', 'chat_input', 1_000_000);\r\n  expect(await policy.limit('test_user', 'chat_input', false)).toBe(1_000_000);\r\n\r\n  const allowed = await policy.allow('test_user', 'chat_input', 750_000);\r\n  expect(allowed).toBe(true);\r\n\r\n  await policy.removeCustomerOverride('test_user', 'chat_input');\r\n  expect(await policy.limit('test_user', 'chat_input', false)).toBe(500_000);\r\n});\n"})}),"\n",(0,i.jsx)(t.hr,{}),"\n",(0,i.jsxs)(t.h2,{id:"difference",children:["Testing policy changes with ",(0,i.jsx)(t.code,{children:"difference()"})]}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.code,{children:"difference()"})," computes a structured diff between two policy instances \u2014 useful for verifying that a policy change affects exactly what you intended and nothing else."]}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-typescript",children:"test('adding enterprise plan does not change starter or growth', async () => {\r\n  const before = await Limitr.new(readFileSync('./policy.yaml', 'utf-8'), 'yaml');\r\n  const after  = await Limitr.new(readFileSync('./policy.new.yaml', 'utf-8'), 'yaml');\r\n\r\n  // before is treated as the schema for the diff\r\n  const diff = await before.difference(after);\r\n\r\n  const changedKeys = Object.keys(diff);\r\n  expect(changedKeys).toContain('enterprise');\r\n  expect(changedKeys).not.toContain('starter');\r\n  expect(changedKeys).not.toContain('growth');\r\n});\n"})}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.code,{children:"difference(other, symmetric?)"})," \u2014 when ",(0,i.jsx)(t.code,{children:"symmetric"})," is ",(0,i.jsx)(t.code,{children:"false"})," (default), only changes relative to the calling policy's structure are returned. When ",(0,i.jsx)(t.code,{children:"true"}),", changes in both directions are included."]}),"\n",(0,i.jsx)(t.hr,{}),"\n",(0,i.jsx)(t.h2,{id:"shipping",children:"What to test before shipping a pricing change"}),"\n",(0,i.jsx)(t.p,{children:"When changing limits, adding a plan, or adjusting credit prices \u2014 not just authoring a policy for the first time \u2014 test the specific change:"}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Limit changes"})," \u2014 Test that the new limit blocks or allows at the right value. Test that existing customers whose meters are already near the old limit behave correctly under the new one."]}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"New plan"})," \u2014 Test that all entitlements gate correctly. Test that a customer on the new plan can't access entitlements from other plans. Test upgrade and downgrade paths to and from the new plan."]}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Credit price changes"})," \u2014 These don't affect enforcement, but verify ",(0,i.jsx)(t.code,{children:"customerMarginSnapshot()"})," returns the expected margin values if your billing code depends on it."]}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Exchange table changes"})," \u2014 Test that ",(0,i.jsx)(t.code,{children:"creditExchange()"})," returns the expected conversion values. Test that grants are drawn correctly at the new exchange rate."]}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"New topup"})," \u2014 Test that ",(0,i.jsx)(t.code,{children:"applyCustomerTopup()"})," creates a grant with the right starting value. Test that the grant is drawn before overage fires. Test expiry if ",(0,i.jsx)(t.code,{children:"expires_after"})," is set."]})]})}function h(e={}){let{wrapper:t}={...(0,s.R)(),...e.components};return t?(0,i.jsx)(t,{...e,children:(0,i.jsx)(d,{...e})}):d(e)}},8453(e,t,n){n.d(t,{R:()=>a,x:()=>o});var r=n(6540);let i={},s=r.createContext(i);function a(e){let t=r.useContext(s);return r.useMemo(function(){return"function"==typeof e?e(t):{...t,...e}},[t,e])}function o(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:a(e.components),r.createElement(s.Provider,{value:t},e.children)}}}]);