https://bastio.com/assets/response-cache-D70HVnyO.js

1import{j as e}from"./index-BkgFiefw.js";let i={title:"Response Cache",description:"Identical-prompt dedup for /v1/chat/completions and Workspace conversations. Cuts upstream LLM cost + latency. Off by default; opt in per-customer."},r=[{href:"#workspace-caching-semantics"},{href:"mailto:[email protected]?subject=Model-scoped%20cache%20invalidation"},{href:"#workspace-caching-semantics"}],o={contents:[{heading:void 0,content:`The Bastio Response Cache stores upstream LLM responses keyed by the
2calling request's fingerprint. When the same prompt comes back, we
3serve the stored response instead of making the API call again. Cuts
4cost (you don't pay upstream twice) and latency (cached replies are
5served in milliseconds, not seconds).`},{heading:void 0,content:`Cloud-only. Off by default — you opt in per-customer when you're
6comfortable with the semantics.`},{heading:"how-it-works",content:"Two surfaces go through the cache:"},{heading:"how-it-works",content:`Gateway — POST /v1/chat/completions, the OpenAI-compatible API
7your customer apps call directly. The cache hashes the verbatim
8request body. Two calls with byte-identical bodies dedupe.`},{heading:"how-it-works",content:`Workspace — POST /v1/workspace/conversations/{id}/messages
9and /messages/stream, the employee chat product. The cache
10hashes the system prompt + the latest user message (history is
11intentionally stripped — see Workspace caching semantics).`},{heading:"how-it-works",content:`Both paths go through the same providers.Client wrapper, so the
12cache covers anything that eventually calls an LLM provider —
13gateway, workspace, branded chat — without each surface needing its
14own integration.`},{heading:"what-goes-into-the-fingerprint",content:`Gateway: the entire HTTP request body. If your client adds
15a user or request_id field that differs per call, the cache
16won't hit — that's deliberate (correctness over hit rate). Stable
17request bodies dedupe; varying bodies don't.`},{heading:"what-goes-into-the-fingerprint",content:`Workspace: system prompt (assistant configuration) + latest
18user message + the typed sampling params (model, temperature,
19top_p, max_tokens). Conversation history is dropped.`},{heading:"what-goes-into-the-fingerprint",content:`Always: the calling customer id and the source ("gateway" vs
20"workspace") are folded into the hash AND used as the Redis key
21prefix, so traffic never crosses tenants or surfaces.`},{heading:"whats-never-cached",content:`Tool-call responses. If the upstream model invokes tools
22(tool_calls or function_call), the response is NOT stored.
23Tool invocations must actually fire; caching them would break
24agentic flows.`},{heading:"whats-never-cached",content:"Non-2xx responses. Upstream errors never poison the cache."},{heading:"whats-never-cached",content:`Empty or malformed responses. A response under ~8 bytes is
25treated as broken and skipped — the customer's first miss won't
26serve a blank reply to everyone else.`},{heading:"whats-never-cached",content:`Non-deterministic requests by default. When temperature > 0
27or top_p < 1, the cache bypasses unless you've turned on
28"Cache non-deterministic responses" (see below).`},{heading:"enabling",content:"Open Cloud Dashboard → Performance → Response Cache."},{heading:"enabling",content:"Toggle Enable response cache ON."},{heading:"enabling",content:`(Optional) Set a TTL — defaults to 1 hour. Range: 60s to 7 days.
29Longer TTL means more savings but staler answers when the
30underlying model is updated.`},{heading:"enabling",content:`(Optional) Add models or routes to the opt-out lists for cases
31where caching would surprise users (RAG models that pull fresh
32context per call, custom proxy routes whose responses must
33always be live).`},{heading:"enabling",content:"Caching starts on the next request."},{heading:"quick-api-check",content:`First call: ~1s, no special headers. Second identical call: returns
34in milliseconds with an X-Bastio-Cache: hit header on the response.`},{heading:"dashboard",content:`The This month widget at the top of the Response Cache page
35shows four numbers, refreshed on every cache event:`},{heading:"dashboard",content:"Hits — requests served from cache (each avoids one upstream call)."}

35,{heading:"dashboard",content:`Misses — cache lookups that didn't find a stored entry. A miss
36is normal on the first call for any unique prompt.`},{heading:"dashboard",content:`Tokens saved — input + output tokens you didn't pay upstream
37for, summed across this month's hits.`},{heading:"dashboard",content:`USD saved — estimated dollar savings, calculated from cached
38tokens × published per-million-token rates per model. Conservative
39when a model isn't in our pricing table — we fall back to a low
40default rate so we never overstate.`},{heading:"workspace-caching-semantics",content:`Workspace conversations grow with every turn. If we hashed the
41whole message array, sending the same user prompt twice in a chat
42would never hit cache — the assistant turn between them makes the
43arrays differ.`},{heading:"workspace-caching-semantics",content:`So workspace caching uses the system prompt + latest user
44message as the fingerprint. Same user prompt across new
45conversations, or repeated within one conversation, dedupes
46correctly.`},{heading:"workspace-caching-semantics",content:`Tradeoff to know about: contextual back-references in workspace
47will cache by their literal text. If a user says "draft me an email
48about onboarding" and then later "make it shorter", the second
49prompt is short and could match an unrelated cached "make it
50shorter" reply from a different conversation, producing a
51less-context-aware response. For independent questions ("what's our
52PTO policy?", "summarize this", "draft an email about X") this is
53exactly what you want. For tight multi-turn workflows, set the TTL
54short or disable workspace caching via the model opt-out list.`},{heading:"non-deterministic-responses",content:`When Cache non-deterministic responses is off (default),
55requests with temperature > 0 or top_p < 1 bypass the cache
56entirely — non-deterministic prompts deserve fresh rolls each time.`},{heading:"non-deterministic-responses",content:`Turn this on only if you explicitly accept that identical prompts
57will return identical "creative" answers for the full TTL window.
58Useful for high-traffic deterministic workloads (FAQ bots,
59extractive summarization, structured-JSON generation) where the
60non-determinism is theoretical and the dedup is real money.`},{heading:"opt-out-lists",content:`Add specific models or request paths that should always bypass the
61cache:`},{heading:"opt-out-lists",content:`Exclude models — match is exact on the model field of the
62request. Useful for RAG models (gpt-4o-rag, claude-haiku-rag)
63whose responses depend on retrieval that happens upstream.`},{heading:"opt-out-lists",content:`Exclude routes — match is exact on the request URL path.
64Useful when you've mounted a custom proxy route whose responses
65must always be live.`},{heading:"manual-flush",content:`The Flush cache button on the Response Cache page drops every
66cached response for your tenant. Use after a model has shipped a
67fix you want everyone to see immediately. The next request for
68each prompt re-fetches from upstream; the cache then repopulates
69normally.`},{heading:"manual-flush",content:`Need to flush only a specific model (e.g. just gpt-4o) without
70nuking the rest? Contact support
71and we'll wire it up.`},{heading:"tenant-isolation",content:`Cache entries are namespaced by (customer_id, source, key_hash).
72Two different customers with byte-identical requests never share a
73cache entry. The customer id is mixed into the SHA-256 digest AND
74used as a Redis key prefix — two independent layers of isolation,
75either of which is sufficient.`},{heading:"whats-not-yet",content:`Semantic / embedding cache. Identical prompts dedupe; similar
76prompts don't. A semantic cache would catch paraphrases ("What's
77France's capital?" vs "Capital of France?") but introduces false
78positives where similar-looking prompts have different correct
79answers. Deliberately not built for v1; the safety-first story
80matters more than maximum hit rate. If a paying customer pushes
81for it, we'll revisit.`},{heading:"whats-not-yet",content:`Hit-rate breakdown dashboard. Today's widget shows monthly
82totals. Per-model, per-route, time-series breakdowns are on the
83backlog — surface when there's signal that customers want to
84tune their cache configs.`},{heading:"whats-not-yet",content:`Model-scoped flush UI. Backend supports it; the dashboard
85button flushes the whole tenant. Email support for now.`},{heading:"will-it-serve-wrong-answers",content:`Not under the default settings. The cache is exact-match by
86fingerprint — different prompts produce different hashes. We
87deliberately do NOT do semantic matching (where similar-looking
88prompts could share an entry);

88 that's the path to false positives.`},{heading:"will-it-serve-wrong-answers",content:`If you turn on "Cache non-deterministic responses", you're
89explicitly accepting that the same prompt with temperature > 0
90will return the same "creative" answer for the TTL window. That's
91a known tradeoff, not a bug.`},{heading:"will-it-serve-wrong-answers",content:`For workspace specifically, see Workspace caching
92semantics — there's one edge case
93worth knowing about.`},{heading:"what-if-my-customer-app-sends-a-unique-field-per-call",content:`If your request body includes user, request_id, or custom
94metadata that varies per call, the gateway cache won't hit because
95the bytes differ. Either remove those fields from the body (Bastio
96already records them separately for traces), or accept lower hit
97rates.`},{heading:"does-workspace-and-gateway-share-cache-entries",content:`No. Same customer, same canonical prompt on workspace vs gateway
98hashes to different keys (the source is mixed into the hash). They
99are explicitly separate namespaces.`},{heading:"how-is-usd-saved-computed",content:`tokens_in_saved × input_rate + tokens_out_saved × output_rate
100per model, summed across this month's hits. Rates are the published
101per-million-token prices from each provider's documentation. When a
102model isn't in our table, we fall back to a conservative rate (so
103we never overstate savings). Updated periodically as providers
104publish new pricing.`},{heading:"can-i-see-which-prompts-are-hitting-cache",content:`Not yet — the dashboard surfaces aggregate stats only. Per-prompt
105drill-down is a Phase 2 surface. Contact support if this is
106blocking for your team and we'll prioritise it.`}],headings:[{id:"how-it-works",content:"How it works"},{id:"what-goes-into-the-fingerprint",content:"What goes into the fingerprint"},{id:"whats-never-cached",content:"What's never cached"},{id:"enabling",content:"Enabling"},{id:"verifying-its-working",content:"Verifying it's working"},{id:"quick-api-check",content:"Quick API check"},{id:"dashboard",content:"Dashboard"},{id:"workspace-caching-semantics",content:"Workspace caching semantics"},{id:"non-deterministic-responses",content:"Non-deterministic responses"},{id:"opt-out-lists",content:"Opt-out lists"},{id:"manual-flush",content:"Manual flush"},{id:"tenant-isolation",content:"Tenant isolation"},{id:"whats-not-yet",content:"What's not (yet)"},{id:"faq",content:"FAQ"},{id:"will-it-serve-wrong-answers",content:"Will it serve wrong answers?"},{id:"what-if-my-customer-app-sends-a-unique-field-per-call",content:"What if my customer app sends a unique field per call?"},{id:"does-workspace-and-gateway-share-cache-entries",content:"Does workspace and gateway share cache entries?"},{id:"how-is-usd-saved-computed",content:'How is "USD saved" computed?'},{id:"can-i-see-which-prompts-are-hitting-cache",content:"Can I see which prompts are hitting cache?"}]};const h=[{depth:2,url:"#how-it-works",title:e.jsx(e.Fragment,{children:"How it works"})},{depth:3,url:"#what-goes-into-the-fingerprint",title:e.jsx(e.Fragment,{children:"What goes into the fingerprint"})},{depth:3,url:"#whats-never-cached",title:e.jsx(e.Fragment,{children:"What's never cached"})},{depth:2,url:"#enabling",title:e.jsx(e.Fragment,{children:"Enabling"})},{depth:2,url:"#verifying-its-working",title:e.jsx(e.Fragment,{children:"Verifying it's working"})},{depth:3,url:"#quick-api-check",title:e.jsx(e.Fragment,{children:"Quick API check"})},{depth:3,url:"#dashboard",title:e.jsx(e.Fragment,{children:"Dashboard"})},{depth:2,url:"#workspace-caching-semantics",title:e.jsx(e.Fragment,{children:"Workspace caching semantics"})},{depth:2,url:"#non-deterministic-responses",title:e.jsx(e.Fragment,{children:"Non-deterministic responses"})},{depth:2,url:"#opt-out-lists",title:e.jsx(e.Fragment,{children:"Opt-out lists"})},{depth:2,url:"#manual-flush",title:e.jsx(e.Fragment,{children:"Manual flush"})},{depth:2,url:"#tenant-isolation",title:e.jsx(e.Fragment,{children:"Tenant isolation"})},{depth:2,url:"#whats-not-yet",title:e.jsx(e.Fragment,{children:"What's not (yet)"})},{depth:2,url:"#faq",title:e.jsx(e.Fragment,{children:"FAQ"})},{depth:3,url:"#will-it-serve-wrong-answers",title:e.jsx(e.Fragment,{children:"Will it serve wrong answers?"})},{depth:3,url:"#what-if-my-customer-app-sends-a-unique-field-per-call",title:e.jsx(e.Fragment,{children:"What if my customer app sends a unique field per 

106call?"})},{depth:3,url:"#does-workspace-and-gateway-share-cache-entries",title:e.jsx(e.Fragment,{children:"Does workspace and gateway share cache entries?"})},{depth:3,url:"#how-is-usd-saved-computed",title:e.jsx(e.Fragment,{children:'How is "USD saved" computed?'})},{depth:3,url:"#can-i-see-which-prompts-are-hitting-cache",title:e.jsx(e.Fragment,{children:"Can I see which prompts are hitting cache?"})}];function n(s){const t={a:"a",code:"code",h2:"h2",h3:"h3",li:"li",ol:"ol",p:"p",pre:"pre",span:"span",strong:"strong",ul:"ul",...s.components};return e.jsxs(e.Fragment,{children:[e.jsx(t.p,{children:`The Bastio Response Cache stores upstream LLM responses keyed by the
107calling request's fingerprint. When the same prompt comes back, we
108serve the stored response instead of making the API call again. Cuts
109cost (you don't pay upstream twice) and latency (cached replies are
110served in milliseconds, not seconds).`}),`
111`,e.jsx(t.p,{children:`Cloud-only. Off by default — you opt in per-customer when you're
112comfortable with the semantics.`}),`
113`,e.jsx(t.h2,{id:"how-it-works",children:"How it works"}),`
114`,e.jsx(t.p,{children:"Two surfaces go through the cache:"}),`
115`,e.jsxs(t.ul,{children:[`
116`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Gateway"})," — ",e.jsx(t.code,{children:"POST /v1/chat/completions"}),`, the OpenAI-compatible API
117your customer apps call directly. The cache hashes the verbatim
118request body. Two calls with byte-identical bodies dedupe.`]}),`
119`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Workspace"})," — ",e.jsx(t.code,{children:"POST /v1/workspace/conversations/{id}/messages"}),`
120and `,e.jsx(t.code,{children:"/messages/stream"}),`, the employee chat product. The cache
121hashes the system prompt + the latest user message (history is
122intentionally stripped — see `,e.jsx(t.a,{href:"#workspace-caching-semantics",children:"Workspace caching semantics"}),")."]}),`
123`]}),`
124`,e.jsxs(t.p,{children:["Both paths go through the same ",e.jsx(t.code,{children:"providers.Client"}),` wrapper, so the
125cache covers anything that eventually calls an LLM provider —
126gateway, workspace, branded chat — without each surface needing its
127own integration.`]}),`
128`,e.jsx(t.h3,{id:"what-goes-into-the-fingerprint",children:"What goes into the fingerprint"}),`
129`,e.jsxs(t.ul,{children:[`
130`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Gateway:"}),` the entire HTTP request body. If your client adds
131a `,e.jsx(t.code,{children:"user"})," or ",e.jsx(t.code,{children:"request_id"}),` field that differs per call, the cache
132won't hit — that's deliberate (correctness over hit rate). Stable
133request bodies dedupe; varying bodies don't.`]}),`
134`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Workspace:"}),` system prompt (assistant configuration) + latest
135user message + the typed sampling params (model, temperature,
136top_p, max_tokens). Conversation history is dropped.`]}),`
137`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Always:"}),` the calling customer id and the source ("gateway" vs
138"workspace") are folded into the hash AND used as the Redis key
139prefix, so traffic never crosses tenants or surfaces.`]}),`
140`]}),`
141`,e.jsx(t.h3,{id:"whats-never-cached",children:"What's never cached"}),`
142`,e.jsxs(t.ul,{children:[`
143`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Tool-call responses."}),` If the upstream model invokes tools
144(`,e.jsx(t.code,{children:"tool_calls"})," or ",e.jsx(t.code,{children:"function_call"}),`), the response is NOT stored.
145Tool invocations must actually fire; caching them would break
146agentic flows.`]}),`
147`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Non-2xx responses."})," Upstream errors never poison the cache."]}),`
148`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Empty or malformed responses."}),` A response under ~8 bytes is
149treated as broken and skipped — the customer's first miss won't
150serve a blank reply to everyone else.`]}),`
151`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Non-deterministic requests by default."})," When ",e.jsx(t.code,{children:"temperature > 0"}),`
152or `,e.jsx(t.code,{children:"top_p < 1"}),`, the cache bypasses unless you've turned on
153"Cache non-deterministic responses" (see below).`]}),`
154`]}),`
155`,e.jsx(t.h2,{id:"enabling",children:"Enabling"}),`
156`,e.jsxs(t.ol,{children:[`
157`,e.jsxs(t.li,{children:["Open ",e.jsx(t.strong,{children:"Cloud Dashboard → Performance → Re

157sponse Cache"}),"."]}),`
158`,e.jsxs(t.li,{children:["Toggle ",e.jsx(t.strong,{children:"Enable response cache"})," ON."]}),`
159`,e.jsx(t.li,{children:`(Optional) Set a TTL — defaults to 1 hour. Range: 60s to 7 days.
160Longer TTL means more savings but staler answers when the
161underlying model is updated.`}),`
162`,e.jsx(t.li,{children:`(Optional) Add models or routes to the opt-out lists for cases
163where caching would surprise users (RAG models that pull fresh
164context per call, custom proxy routes whose responses must
165always be live).`}),`
166`]}),`
167`,e.jsx(t.p,{children:"Caching starts on the next request."}),`
168`,e.jsx(t.h2,{id:"verifying-its-working",children:"Verifying it's working"}),`
169`,e.jsx(t.h3,{id:"quick-api-check",children:"Quick API check"}),`
170`,e.jsx(e.Fragment,{children:e.jsx(t.pre,{className:"shiki shiki-themes github-light github-dark",style:{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},tabIndex:"0",icon:'<svg viewBox="0 0 24 24"><path d="m 4,4 a 1,1 0 0 0 -0.7070312,0.2929687 1,1 0 0 0 0,1.4140625 L 8.5859375,11 3.2929688,16.292969 a 1,1 0 0 0 0,1.414062 1,1 0 0 0 1.4140624,0 l 5.9999998,-6 a 1.0001,1.0001 0 0 0 0,-1.414062 L 4.7070312,4.2929687 A 1,1 0 0 0 4,4 Z m 8,14 a 1,1 0 0 0 -1,1 1,1 0 0 0 1,1 h 8 a 1,1 0 0 0 1,-1 1,1 0 0 0 -1,-1 z" fill="currentColor" /></svg>',children:e.jsxs(t.code,{children:[e.jsxs(t.span,{className:"line",children:[e.jsx(t.span,{style:{"--shiki-light":"#6F42C1","--shiki-dark":"#B392F0"},children:"curl"}),e.jsx(t.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" -i"}),e.jsx(t.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" -X"}),e.jsx(t.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:" POST"}),e.jsx(t.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:" https://api.bastio.com/v1/chat/completions"}),e.jsx(t.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" \\"})]}),`
171`,e.jsxs(t.span,{className:"line",children:[e.jsx(t.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:"  -H"}),e.jsx(t.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "Authorization: Bearer '}),e.jsx(t.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:"$YOUR_KEY"}),e.jsx(t.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:'"'}),e.jsx(t.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" \\"})]}),`
172`,e.jsxs(t.span,{className:"line",children:[e.jsx(t.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:"  -H"}),e.jsx(t.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "Content-Type: application/json"'}),e.jsx(t.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" \\"})]}),`
173`,e.jsxs(t.span,{className:"line",children:[e.jsx(t.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:"  -d"}),e.jsx(t.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:` '{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}'`})]})]})})}),`
174`,e.jsxs(t.p,{children:[`First call: ~1s, no special headers. Second identical call: returns
175in milliseconds with an `,e.jsx(t.code,{children:"X-Bastio-Cache: hit"})," header on the response."]}),`
176`,e.jsx(t.h3,{id:"dashboard",children:"Dashboard"}),`
177`,e.jsxs(t.p,{children:["The ",e.jsx(t.strong,{children:"This month"}),` widget at the top of the Response Cache page
178shows four numbers, refreshed on every cache event:`]}),`
179`,e.jsxs(t.ul,{children:[`
180`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Hits"})," — requests served from cache (each avoids one upstream call)."]}),`
181`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Misses"}),` — cache lookups that didn't find a stored entry. A miss
182is normal on the first call for any unique prompt.`]}),`
183`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Tokens saved"}),` — input + output tokens you didn't pay upstream
184for, summed across this month's hits.`]}),`
185`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"USD saved"}

185),` — estimated dollar savings, calculated from cached
186tokens × published per-million-token rates per model. Conservative
187when a model isn't in our pricing table — we fall back to a low
188default rate so we never overstate.`]}),`
189`]}),`
190`,e.jsx(t.h2,{id:"workspace-caching-semantics",children:"Workspace caching semantics"}),`
191`,e.jsx(t.p,{children:`Workspace conversations grow with every turn. If we hashed the
192whole message array, sending the same user prompt twice in a chat
193would never hit cache — the assistant turn between them makes the
194arrays differ.`}),`
195`,e.jsxs(t.p,{children:["So workspace caching uses the ",e.jsx(t.strong,{children:`system prompt + latest user
196message`}),` as the fingerprint. Same user prompt across new
197conversations, or repeated within one conversation, dedupes
198correctly.`]}),`
199`,e.jsxs(t.p,{children:[e.jsx(t.strong,{children:"Tradeoff to know about:"}),` contextual back-references in workspace
200will cache by their literal text. If a user says "draft me an email
201about onboarding" and then later "make it shorter", the second
202prompt is short and could match an unrelated cached "make it
203shorter" reply from a different conversation, producing a
204less-context-aware response. For independent questions ("what's our
205PTO policy?", "summarize this", "draft an email about X") this is
206exactly what you want. For tight multi-turn workflows, set the TTL
207short or disable workspace caching via the model opt-out list.`]}),`
208`,e.jsx(t.h2,{id:"non-deterministic-responses",children:"Non-deterministic responses"}),`
209`,e.jsxs(t.p,{children:["When ",e.jsx(t.strong,{children:"Cache non-deterministic responses"})," is ",e.jsx(t.strong,{children:"off"}),` (default),
210requests with `,e.jsx(t.code,{children:"temperature > 0"})," or ",e.jsx(t.code,{children:"top_p < 1"}),` bypass the cache
211entirely — non-deterministic prompts deserve fresh rolls each time.`]}),`
212`,e.jsx(t.p,{children:`Turn this on only if you explicitly accept that identical prompts
213will return identical "creative" answers for the full TTL window.
214Useful for high-traffic deterministic workloads (FAQ bots,
215extractive summarization, structured-JSON generation) where the
216non-determinism is theoretical and the dedup is real money.`}),`
217`,e.jsx(t.h2,{id:"opt-out-lists",children:"Opt-out lists"}),`
218`,e.jsx(t.p,{children:`Add specific models or request paths that should always bypass the
219cache:`}),`
220`,e.jsxs(t.ul,{children:[`
221`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Exclude models"})," — match is exact on the ",e.jsx(t.code,{children:"model"}),` field of the
222request. Useful for RAG models (`,e.jsx(t.code,{children:"gpt-4o-rag"}),", ",e.jsx(t.code,{children:"claude-haiku-rag"}),`)
223whose responses depend on retrieval that happens upstream.`]}),`
224`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Exclude routes"}),` — match is exact on the request URL path.
225Useful when you've mounted a custom proxy route whose responses
226must always be live.`]}),`
227`]}),`
228`,e.jsx(t.h2,{id:"manual-flush",children:"Manual flush"}),`
229`,e.jsxs(t.p,{children:["The ",e.jsx(t.strong,{children:"Flush cache"}),` button on the Response Cache page drops every
230cached response for your tenant. Use after a model has shipped a
231fix you want everyone to see immediately. The next request for
232each prompt re-fetches from upstream; the cache then repopulates
233normally.`]}),`
234`,e.jsxs(t.p,{children:["Need to flush only a specific model (e.g. just ",e.jsx(t.code,{children:"gpt-4o"}),`) without
235nuking the rest? `,e.jsx(t.a,{href:"mailto:[email protected]?subject=Model-scoped%20cache%20invalidation",children:"Contact support"}),`
236and we'll wire it up.`]}),`
237`,e.jsx(t.h2,{id:"tenant-isolation",children:"Tenant isolation"}),`
238`,e.jsxs(t.p,{children:["Cache entries are namespaced by ",e.jsx(t.code,{children:"(customer_id, source, key_hash)"}),`.
239Two different customers with byte-identical requests never share a
240cache entry. The customer id is mixed into the SHA-256 digest AND
241used as a Redis key prefix — two independent layers of isolation,
242either of which is sufficient.`]}),`
243`,e.jsx(t.h2,{id:"whats-not-yet",children:"What's not (yet)"}),`
244`,e.jsxs(t.ul,{children:[`
245`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Semantic / embedding cache."}),` Identical prompts dedupe; similar
246prompts don't. A semantic cache would catch paraphrases ("What's
247France's capital?" vs "Capital of France?") but introduces false

248positives where similar-looking prompts have different correct
249answers. Deliberately not built for v1; the safety-first story
250matters more than maximum hit rate. If a paying customer pushes
251for it, we'll revisit.`]}),`
252`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Hit-rate breakdown dashboard."}),` Today's widget shows monthly
253totals. Per-model, per-route, time-series breakdowns are on the
254backlog — surface when there's signal that customers want to
255tune their cache configs.`]}),`
256`,e.jsxs(t.li,{children:[e.jsx(t.strong,{children:"Model-scoped flush UI."}),` Backend supports it; the dashboard
257button flushes the whole tenant. Email support for now.`]}),`
258`]}),`
259`,e.jsx(t.h2,{id:"faq",children:"FAQ"}),`
260`,e.jsx(t.h3,{id:"will-it-serve-wrong-answers",children:"Will it serve wrong answers?"}),`
261`,e.jsx(t.p,{children:`Not under the default settings. The cache is exact-match by
262fingerprint — different prompts produce different hashes. We
263deliberately do NOT do semantic matching (where similar-looking
264prompts could share an entry); that's the path to false positives.`}),`
265`,e.jsxs(t.p,{children:[`If you turn on "Cache non-deterministic responses", you're
266explicitly accepting that the same prompt with `,e.jsx(t.code,{children:"temperature > 0"}),`
267will return the same "creative" answer for the TTL window. That's
268a known tradeoff, not a bug.`]}),`
269`,e.jsxs(t.p,{children:["For workspace specifically, see ",e.jsx(t.a,{href:"#workspace-caching-semantics",children:`Workspace caching
270semantics`}),` — there's one edge case
271worth knowing about.`]}),`
272`,e.jsx(t.h3,{id:"what-if-my-customer-app-sends-a-unique-field-per-call",children:"What if my customer app sends a unique field per call?"}),`
273`,e.jsxs(t.p,{children:["If your request body includes ",e.jsx(t.code,{children:"user"}),", ",e.jsx(t.code,{children:"request_id"}),`, or custom
274metadata that varies per call, the gateway cache won't hit because
275the bytes differ. Either remove those fields from the body (Bastio
276already records them separately for traces), or accept lower hit
277rates.`]}),`
278`,e.jsx(t.h3,{id:"does-workspace-and-gateway-share-cache-entries",children:"Does workspace and gateway share cache entries?"}),`
279`,e.jsx(t.p,{children:`No. Same customer, same canonical prompt on workspace vs gateway
280hashes to different keys (the source is mixed into the hash). They
281are explicitly separate namespaces.`}),`
282`,e.jsx(t.h3,{id:"how-is-usd-saved-computed",children:'How is "USD saved" computed?'}),`
283`,e.jsxs(t.p,{children:[e.jsx(t.code,{children:"tokens_in_saved × input_rate + tokens_out_saved × output_rate"}),`
284per model, summed across this month's hits. Rates are the published
285per-million-token prices from each provider's documentation. When a
286model isn't in our table, we fall back to a conservative rate (so
287we never overstate savings). Updated periodically as providers
288publish new pricing.`]}),`
289`,e.jsx(t.h3,{id:"can-i-see-which-prompts-are-hitting-cache",children:"Can I see which prompts are hitting cache?"}),`
290`,e.jsx(t.p,{children:`Not yet — the dashboard surfaces aggregate stats only. Per-prompt
291drill-down is a Phase 2 surface. Contact support if this is
292blocking for your team and we'll prioritise it.`})]})}function c(s={}){const{wrapper:t}=s.components||{};return t?e.jsx(t,{...s,children:e.jsx(n,{...s})}):n(s)}export{c as default,r as extractedReferences,i as frontmatter,o as structuredData,h as toc};