1import{j as e}from"./index-BkgFiefw.js";let l={title:"Ollama",description:"Route local Ollama models through Bastio for self-hosted or on-prem AI."},a=[{href:"https://ollama.com"},{href:"/docs/developer-apis/detect"},{href:"/docs/getting-started"},{href:"https://ollama.com/library"},{href:"/docs/observability"},{href:"/docs/threats"},{href:"/docs/architecture"}],r={contents:[{heading:void 0,content:`Ollama runs LLMs on your own hardware â 2laptops, servers, on-prem clusters. Bastio in front of Ollama 3gives you the same security scanning and audit trail you get for 4OpenAI / Anthropic, without any data ever leaving your network.`},{heading:void 0,content:"This is the right setup for:"},{heading:void 0,content:"Air-gapped or regulated environments"},{heading:void 0,content:"Cost-sensitive prototyping (no token bills)"},{heading:void 0,content:"Latency-critical edge deployments"},{heading:void 0,content:`Anyone who wants to evaluate Bastio without provider account 5setup`},{heading:void 0,content:"The repository includes an Ollama client, but the public gateway's model router currently falls back unknown model names to OpenAI. Until explicit Ollama routing is added, scan with /v1/detect and call Ollama directly."},{heading:"prerequisites",content:`An Ollama install running somewhere reachable from your Bastio 6gateway. Default port: 11434.`},{heading:"prerequisites",content:`At least one model pulled (ollama pull llama3.3, 7ollama pull mistral-small).`},{heading:"prerequisites",content:"A running Bastio gateway (Getting started)."},{heading:"no-api-key-needed",content:"Ollama doesn't authenticate. The proxy just needs the URL."},{heading:"create-the-proxy",content:"Configure the Ollama endpoint in the server deployment or provider settings. The proxy management request stores routing metadata, not provider credentials or endpoints."},{heading:"send-a-request",content:"The model name matches what ollama list shows on your install."},{heading:"supported-features",content:"Chat completions â full surface that Ollama exposes."},{heading:"supported-features",content:"Streaming â stream: true works."},{heading:"supported-features",content:`Tool calling â supported on models that advertise it 8(Llama 3.1+, Mistral Large, Qwen 2.5+). Bastio passes the 9tools field through; Ollama returns tool_calls in the 10same OpenAI-compatible shape.`},{heading:"supported-features",content:`Multimodal â vision models (llava, llama3.2-vision) 11accept image_url blocks.`},{heading:"supported-features",content:"Embeddings are not exposed by the Bastio gateway route today."},{heading:"models",content:`Ollama's catalog is huge â anything in 12ollama.com/library works. Common 13picks:`},{heading:"models",content:"Use case"},{heading:"models",content:"Model"},{heading:"models",content:"General chat (large)"},{heading:"models",content:"llama3.3:70b, qwen2.5:72b"},{heading:"models",content:"General chat (small)"},{heading:"models",content:"llama3.2:3b, phi-4"},{heading:"models",content:"Code"},{heading:"models",content:"qwen2.5-coder:32b, deepseek-coder-v2"},{heading:"models",content:"Embeddings"},{heading:"models",content:"nomic-embed-text, mxbai-embed-large"},{heading:"models",content:"Vision"},{heading:"models",content:"llama3.2-vision:11b, llava:34b"},{heading:"models",content:`Run ollama pull <name> to fetch; the model becomes available 14to Bastio immediately on the next request.`},{heading:"hardware-sizing",content:"Model size"},{heading:"hardware-sizing",content:"Min VRAM"},{heading:"hardware-sizing",content:"Notes"},{heading:"hardware-sizing",content:"3B / 7B"},{heading:"hardware-sizing",content:"8 GB"},{heading:"hardware-sizing",content:"Fits a desktop GPU or a Mac with unified memory"},{heading:"hardware-sizing",content:"13B"},{heading:"hardware-sizing",content:"16 GB"},{heading:"hardware-sizing",content:"Mid-range workstation"},{heading:"hardware-sizing",content:"33B / 34B"},{heading:"hardware-sizing",content:"24 GB"},{heading:"hardware-sizing",content:"RTX 3090 / 4090, M2 Max+"},{heading:"hardware-sizing",content:"70B+"},{heading:"hardware-sizing",content:"48 GB+"},{heading:"hardware-sizing",content:"Dual GPU, M2 Ultra, or H100"},{heading:"hardware-sizing",content:`Ollama runs on CPU when there's no GPU, but a 70B model on CPU 15takes ~5-10s per token â fine for batch jobs, painful 16interactively.`},{heading:"gotchas",content:`Cold-start delay. First request after a restart loads the 17model into memory â can take 30+ seconds for a 70B. Bastio 18doesn't time out (default Bastio timeout is 60s, configurable 19via BASTIO_PROVIDER_TIMEOUT), but your client might.`},{heading:"gotchas",content:`Concurrent requests are serialized per-model. Ollama's 20default config holds one model at a time per GPU. Multiple 21proxies can hit different models, but the same model handles 22one inference at a time. Set OLLAMA_NUM_PARALLEL and 23OLLAMA_MAX_LOADED_MODELS if you need concurrency.`},{heading:"gotchas",content:`No native rate limits. Ollama doesn't reject excess 24traffic â it just queues. Bastio's per-proxy 25rate_limit_rpm is the only ceiling. Set it conservatively 26for shared GPUs.`},{heading:"gotchas",content:`Model names include the tag. llama3.3 and llama3.3:70b 27resolve to different things if you have multiple tags pulled. 28Be explicit about the tag in production.`},{heading:"gotchas",content:`Tool-calling format varies by model. Llama 3.x uses one 29format, Qwen another. Test before relying on tools across 30models.`},{heading:"pairing-with-bastio-cloud",content:`Even if you run inference on Ollama, you can still send the 31trace metadata up to Bastio Cloud for centralized observability â 32the gateway emits the same trace records regardless of where 33the model lives. Set CLICKHOUSE_URL to your cloud endpoint 34and analytics flow up; your prompts and responses can stay 35fully on-prem.`},{heading:"where-the-request-shows-up-in-traces",content:`Ollama requests land in Traces with provider: ollama. Cost is 36recorded as 0 â Ollama is unmetered. Token counts come from 37Ollama's response metadata. See Observability.`}
37,{heading:"next",content:`Threats â local models still benefit from 38prompt-injection + secret scanning, especially for 39code-generation workflows.`},{heading:"next",content:`Architecture â Bastio + Ollama runs 40fully on-prem; the only off-host traffic is what you 41intentionally send to ClickHouse for analytics.`}],headings:[{id:"prerequisites",content:"Prerequisites"},{id:"no-api-key-needed",content:"No API key needed"},{id:"create-the-proxy",content:"Create the proxy"},{id:"send-a-request",content:"Send a request"},{id:"python",content:"Python"},{id:"supported-features",content:"Supported features"},{id:"models",content:"Models"},{id:"hardware-sizing",content:"Hardware sizing"},{id:"gotchas",content:"Gotchas"},{id:"pairing-with-bastio-cloud",content:"Pairing with Bastio Cloud"},{id:"where-the-request-shows-up-in-traces",content:"Where the request shows up in traces"},{id:"next",content:"Next"}]};const h=[{depth:2,url:"#prerequisites",title:e.jsx(e.Fragment,{children:"Prerequisites"})},{depth:2,url:"#no-api-key-needed",title:e.jsx(e.Fragment,{children:"No API key needed"})},{depth:2,url:"#create-the-proxy",title:e.jsx(e.Fragment,{children:"Create the proxy"})},{depth:2,url:"#send-a-request",title:e.jsx(e.Fragment,{children:"Send a request"})},{depth:3,url:"#python",title:e.jsx(e.Fragment,{children:"Python"})},{depth:2,url:"#supported-features",title:e.jsx(e.Fragment,{children:"Supported features"})},{depth:2,url:"#models",title:e.jsx(e.Fragment,{children:"Models"})},{depth:2,url:"#hardware-sizing",title:e.jsx(e.Fragment,{children:"Hardware sizing"})},{depth:2,url:"#gotchas",title:e.jsx(e.Fragment,{children:"Gotchas"})},{depth:2,url:"#pairing-with-bastio-cloud",title:e.jsx(e.Fragment,{children:"Pairing with Bastio Cloud"})},{depth:2,url:"#where-the-request-shows-up-in-traces",title:e.jsx(e.Fragment,{children:"Where the request shows up in traces"})},{depth:2,url:"#next",title:e.jsx(e.Fragment,{children:"Next"})}];function n(s){const i={a:"a",blockquote:"blockquote",code:"code",h2:"h2",h3:"h3",li:"li",p:"p",pre:"pre",span:"span",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...s.components};return e.jsxs(e.Fragment,{children:[e.jsxs(i.p,{children:[e.jsx(i.a,{href:"https://ollama.com",children:"Ollama"}),` runs LLMs on your own hardware â 42laptops, servers, on-prem clusters. Bastio in front of Ollama 43gives you the same security scanning and audit trail you get for 44OpenAI / Anthropic, without any data ever leaving your network.`]}),` 45`,e.jsx(i.p,{children:"This is the right setup for:"}),` 46`,e.jsxs(i.ul,{children:[` 47`,e.jsx(i.li,{children:"Air-gapped or regulated environments"}),` 48`,e.jsx(i.li,{children:"Cost-sensitive prototyping (no token bills)"}),` 49`,e.jsx(i.li,{children:"Latency-critical edge deployments"}),` 50`,e.jsx(i.li,{children:`Anyone who wants to evaluate Bastio without provider account 51setup`}),` 52`]}),` 53`,e.jsxs(i.blockquote,{children:[` 54`,e.jsxs(i.p,{children:["The repository includes an Ollama client, but the public gateway's model router currently falls back unknown model names to OpenAI. Until explicit Ollama routing is added, scan with ",e.jsx(i.a,{href:"/docs/developer-apis/detect",children:e.jsx(i.code,{children:"/v1/detect"})})," and call Ollama directly."]}),` 55`]}),` 56`,e.jsx(i.h2,{id:"prerequisites",children:"Prerequisites"}),` 57`,e.jsxs(i.ul,{children:[` 58`,e.jsx(i.li,{children:`An Ollama install running somewhere reachable from your Bastio 59gateway. Default port: 11434.`}),` 60`,e.jsxs(i.li,{children:["At least one model pulled (",e.jsx(i.code,{children:"ollama pull llama3.3"}),`, 61`,e.jsx(i.code,{children:"ollama pull mistral-small"}),")."]}),` 62`,e.jsxs(i.li,{children:["A running Bastio gateway (",e.jsx(i.a,{href:"/docs/getting-started",children:"Getting started"}),")."]}),` 63`]}),` 64`,e.jsx(i.h2,{id:"no-api-key-needed",children:"No API key needed"}),` 65`,e.jsx(i.p,{children:"Ollama doesn't authenticate. The proxy just needs the URL."}),` 66`,e.jsx(i.h2,{id:"create-the-proxy",children:"Create the proxy"}),` 67`,e.jsx(e.Fragment,{children:e.jsx(i.pre,{className:"shiki shiki-themes g
67ithub-light github-dark",style:{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},tabIndex:"0",icon:'<svg viewBox="0 0 24 24"><path d="m 4,4 a 1,1 0 0 0 -0.7070312,0.2929687 1,1 0 0 0 0,1.4140625 L 8.5859375,11 3.2929688,16.292969 a 1,1 0 0 0 0,1.414062 1,1 0 0 0 1.4140624,0 l 5.9999998,-6 a 1.0001,1.0001 0 0 0 0,-1.414062 L 4.7070312,4.2929687 A 1,1 0 0 0 4,4 Z m 8,14 a 1,1 0 0 0 -1,1 1,1 0 0 0 1,1 h 8 a 1,1 0 0 0 1,-1 1,1 0 0 0 -1,-1 z" fill="currentColor" /></svg>',children:e.jsxs(i.code,{children:[e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#6F42C1","--shiki-dark":"#B392F0"},children:"curl"}),e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" -X"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:" POST"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:" http://localhost:4000/v1/proxies"}),e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" \\"})]}),` 68`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" -H"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "Content-Type: application/json"'}),e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" \\"})]}),` 69`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" -d"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:" '{"})]}),` 70`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "name": "local-ollama",'})}),` 71`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "slug": "local-ollama",'})}),` 72`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "target_provider": "ollama",'})}),` 73`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "target_model": "llama3.3:70b"'})}),` 74`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:" }'"})})]})})}),` 75`,e.jsx(i.p,{children:"Configure the Ollama endpoint in the server deployment or provider settings. The proxy management request stores routing metadata, not provider credentials or endpoints."}),` 76`,e.jsx(i.h2,{id:"send-a-request",children:"Send a request"}),` 77`,e.jsxs(i.p,{children:["The model name matches what ",e.jsx(i.code,{children:"ollama list"})," shows on your install."]}),` 78`,e.jsx(e.Fragment,{children:e.jsx(i.pre,{className:"shiki shiki-themes github-light github-dark",style:{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},tabIndex:"0",icon:'<svg viewBox="0 0 24 24"><path d="m 4,4 a 1,1 0 0 0 -0.7070312,0.2929687 1,1 0 0 0 0,1.4140625 L 8.5859375,11 3.2929688,16.292969 a 1,1 0 0 0 0,1.414062 1,1 0 0 0 1.4140624,0 l 5.9999998,-6 a 1.0001,1.0001 0 0 0 0,-1.414062 L 4.7070312,4.2929687 A 1,1 0 0 0 4,4 Z m 8,14 a 1,1 0 0 0 -1,1 1,1 0 0 0 1,1 h 8 a 1,1 0 0 0 1,-1 1,1 0 0 0 -1,-1 z" fill="currentColor" /></svg>',children:e.jsxs(i.code,{children:[e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#6F42C1","--shiki-dark":"#B392F0"},children:"curl"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:" http://localhost:4000/v1/chat/completions"}),e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" \\"})]}),` 79`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" -H"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "Authorization: Bearer '}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:"$BASTIO_GATEWAY_KEY"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:'"'}),e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" \\"})]}),` 80`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" -H"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "Content-Type: application/json"'}),e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" \\"})]}),` 81`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#005CC5","--shiki-dark":"#79B8FF"},children:" -d"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:" '{"})]}),` 82`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "model": "llama3.3:70b",'})}),` 83`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:' "messages": [{"role": "user", "content": "Hello"}]'})}),` 84`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:" }'"})})]})})}),` 85`,e.jsx(i.h3,{id:"python",children:"Python"}),` 86`,e.jsx(e.Fragment,{children:e.jsx(i.pre,{className:"shiki shiki-themes g
86ithub-light github-dark",style:{"--shiki-light":"#24292e","--shiki-dark":"#e1e4e8","--shiki-light-bg":"#fff","--shiki-dark-bg":"#24292e"},tabIndex:"0",icon:'<svg viewBox="0 0 24 24"><path d="M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z" fill="currentColor" /></svg>',children:e.jsxs(i.code,{children:[e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},children:"from"}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:" openai "}),e.jsx(i.span,{style:{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},children:"import"}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:" OpenAI"})]}),` 87`,e.jsx(i.span,{className:"line"}),` 88`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:"client "}),e.jsx(i.span,{style:{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},children:"="}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:" OpenAI("})]}),` 89`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#E36209","--shiki-dark":"#FFAB70"},children:" base_url"}),e.jsx(i.span,{style:{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},children:"="}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:'"http://localhost:4000/v1"'}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:","})]}),` 90`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#E36209","--shiki-dark":"#FFAB70"},children:" api_key"}),e.jsx(i.span,{style:{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},children:"="}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:"os.environ["}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:'"BASTIO_GATEWAY_KEY"'}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:"],"})]}),` 91`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:")"})}),` 92`,e.jsx(i.span,{className:"line"}),` 93`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:"resp "}),e.jsx(i.span,{style:{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},children:"="}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:" client.chat.completions.create("})]}),` 94`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#E36209","--shiki-dark":"#FFAB70"},children:" model"}),e.jsx(i.span,{style:{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},children:"="}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:'"llama3.3:70b"'}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:","})]}),` 95`,e.jsxs(i.span,{className:"line",children:[e.jsx(i.span,{style:{"--shiki-light":"#E36209","--shiki-dark":"#FFAB
9570"},children:" messages"}),e.jsx(i.span,{style:{"--shiki-light":"#D73A49","--shiki-dark":"#F97583"},children:"="}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:"[{"}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:'"role"'}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:": "}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:'"user"'}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:", "}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:'"content"'}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:": "}),e.jsx(i.span,{style:{"--shiki-light":"#032F62","--shiki-dark":"#9ECBFF"},children:'"Hello"'}),e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:"}],"})]}),` 96`,e.jsx(i.span,{className:"line",children:e.jsx(i.span,{style:{"--shiki-light":"#24292E","--shiki-dark":"#E1E4E8"},children:")"})})]})})}),` 97`,e.jsx(i.h2,{id:"supported-features",children:"Supported features"}),` 98`,e.jsxs(i.ul,{children:[` 99`,e.jsxs(i.li,{children:[e.jsx(i.strong,{children:"Chat completions"})," â full surface that Ollama exposes."]}),` 100`,e.jsxs(i.li,{children:[e.jsx(i.strong,{children:"Streaming"})," â ",e.jsx(i.code,{children:"stream: true"})," works."]}),` 101`,e.jsxs(i.li,{children:[e.jsx(i.strong,{children:"Tool calling"}),` â supported on models that advertise it 102(Llama 3.1+, Mistral Large, Qwen 2.5+). Bastio passes the 103`,e.jsx(i.code,{children:"tools"})," field through; Ollama returns ",e.jsx(i.code,{children:"tool_calls"}),` in the 104same OpenAI-compatible shape.`]}),` 105`,e.jsxs(i.li,{children:[e.jsx(i.strong,{children:"Multimodal"})," â vision models (",e.jsx(i.code,{children:"llava"}),", ",e.jsx(i.code,{children:"llama3.2-vision"}),`) 106accept `,e.jsx(i.code,{children:"image_url"})," blocks."]}),` 107`,e.jsx(i.li,{children:"Embeddings are not exposed by the Bastio gateway route today."}),` 108`]}),` 109`,e.jsx(i.h2,{id:"models",children:"Models"}),` 110`,e.jsxs(i.p,{children:[`Ollama's catalog is huge â anything in 111`,e.jsx(i.a,{href:"https://ollama.com/library",children:"ollama.com/library"}),` works. Common 112picks:`]}),` 113`,e.jsxs(i.table,{children:[e.jsx(i.thead,{children:e.jsxs(i.tr,{children:[e.jsx(i.th,{children:"Use case"}),e.jsx(i.th,{children:"Model"})]})}),e.jsxs(i.tbody,{children:[e.jsxs(i.tr,{children:[e.jsx(i.td,{children:"General chat (large)"}),e.jsxs(i.td,{children:[e.jsx(i.code,{children:"llama3.3:70b"}),", ",e.jsx(i.code,{children:"qwen2.5:72b"})]})]}),e.jsxs(i.tr,{children:[e.jsx(i.td,{children:"General chat (small)"}),e.jsxs(i.td,{children:[e.jsx(i.code,{children:"llama3.2:3b"}),", ",e.jsx(i.code,{children:"phi-4"})]})]}),e.jsxs(i.tr,{children:[e.jsx(i.td,{children:"Code"}),e.jsxs(i.td,{children:[e.jsx(i.code,{children:"qwen2.5-coder:32b"}),", ",e.jsx(i.code,{children:"deepseek-coder-v2"})]})]}),e.jsxs(i.tr,{children:[e.jsx(i.td,{children:"Embeddings"}),e.jsxs(i.td,{children:[e.jsx(i.code,{children:"nomic-embed-text"}),", ",e.jsx(i.code,{children:"mxbai-embed-large"})]})]}),e.jsxs(i.tr,{children:[e.jsx(i.td,{children:"Vision"}),e.jsxs(i.td,{children:[e.jsx(i.code,{children:"llama3.2-vision:11b"}),", ",e.jsx(i.code,{children:"llava:34b"})]})]})]})]}),` 114`,e.jsxs(i.p,{children:["Run ",e.jsx(i.code,{children:"ollama pull <name>"}),` to fetch; the model becomes available 115to Bastio immediately on the next request.`]}),` 116`,e.jsx(i.h2,{id:"hardware-sizing",children:"Hardware sizing"}),` 117`,e.jsxs(i.table,{children:[e.jsx(i.thead,{children:e.jsxs(i.tr,{children:[e.jsx(i.th,{children:"Model size"}),e.jsx(i.th,{children:"Min VRAM"}),e.jsx(i.th,{children:"Notes"})]})}),e.jsxs(i.tbody,{children:[e.jsxs(i.tr,{children:[e.jsx(i.td,{children:"3B / 7B"}),e.jsx(i.td,{children:"8 GB"}),e.jsx(i.td,{children:"Fits a desktop GPU or a Mac with unified memory"})]}),e.jsxs(i.tr,{children:[e.jsx(i.td,{children:"13B"}),e.jsx(i.td,{children:"16 GB"}),e.jsx(i.td,{children:"Mid-range workstation"})]}),e.jsxs(i.tr,{children:[e.jsx(i.td,{children:"33B / 34B"}),e.jsx(i.td,{children:"24 GB"}),e.jsx(i.td,{children:"RTX 3090 / 4090, M2 Max+"})]}),e.jsxs(i.tr,{children:[e.jsx(i.td,{children:"70B+"}),e.jsx(i.td,{children:"48 GB+"}),e.jsx(i.td,{children:"Dual GPU, M2 Ultra, or H100"})]})]})]}),` 118`,e.jsx(i.p,{children:`Ollama runs on CPU when there's no GPU, but a 70B model on CPU 119takes ~5-10s per token â fine for batch jobs, painful 120interactively.`}),` 121`,e.jsx(i.h2,{id:"gotchas",children:"Gotchas"}),` 122`,e.jsxs(i.ul,{children:[` 123`,e.jsxs(i.li,{children:[e.jsx(i.strong,{children:"Cold-start delay."}),` First request after a restart loads the 124model into memory â can take 30+ seconds for a 70B. Bastio 125doesn't time out (default Bastio timeout is 60s, configurable 126via `,e.jsx(i.code,{children:"BASTIO_PROVIDER_TIMEOUT"}),"), but your client might."]}),` 127`,e.jsxs(i.li,{children:[e.jsx(i.strong,{children:"Concurrent requests are serialized per-model."}),` Ollama's 128default config holds one model at a time per GPU. Multiple 129proxies can hit different models, but the same model handles 130one inference at a time. Set `,e.jsx(i.code,{children:"OLLAMA_NUM_PARALLEL"}),` and 131`,e.jsx(i.code,{children:"OLLAMA_MAX_LOADED_MODELS"})," if you need concurrency."]}),` 132`,e.jsxs(i.li,{children:[e.jsx(i.strong,{children:"No native rate limits."}),` Ollama doesn't reject excess 133traffic â it just queues. Bastio's per-proxy 134`,e.jsx(i.code,{children:"rate_limit_rpm"}),` is the only ceiling. Set it conservatively 135for shared GPUs.`]}),` 136`,e.jsxs(i.li,{children:[e.jsx(i.strong,{children:"Model names include the tag."})," ",e.jsx(i.code,{children:"llama3.3"})," and ",e.jsx(i.code,{children:"llama3.3:70b"}),` 137resolve to different things if you have multiple tags pulled. 138Be explicit about the tag in production.`]}),` 139`,e.jsxs(i.li,{children:[e.jsx(i.strong,{children:"Tool-calling format varies by model."}),` Llama 3.x uses one 140format, Qwen another. Test before relying on tools across 141models.`]}),` 142`]}),` 143`,e.jsx(i.h2,{id:"pairing-with-bastio-cloud",children:"Pairing with Bastio Cloud"}),` 144`,e.jsxs(i.p,{children:[`Even if you run inference on Ollama, you can still send the 145trace metadata up to Bastio Cloud for centralized observability â 146the gateway emits the same trace records regardless of where 147the model lives. Set `,e.jsx(i.code,{children:"CLICKHOUSE_URL"}),` to your cloud endpoint 148and analytics flow up; your prompts and responses can stay 149fully on-prem.`]}
149),` 150`,e.jsx(i.h2,{id:"where-the-request-shows-up-in-traces",children:"Where the request shows up in traces"}),` 151`,e.jsxs(i.p,{children:["Ollama requests land in Traces with ",e.jsx(i.code,{children:"provider: ollama"}),`. Cost is 152recorded as 0 â Ollama is unmetered. Token counts come from 153Ollama's response metadata. See `,e.jsx(i.a,{href:"/docs/observability",children:"Observability"}),"."]}),` 154`,e.jsx(i.h2,{id:"next",children:"Next"}),` 155`,e.jsxs(i.ul,{children:[` 156`,e.jsxs(i.li,{children:[e.jsx(i.a,{href:"/docs/threats",children:"Threats"}),` â local models still benefit from 157prompt-injection + secret scanning, especially for 158code-generation workflows.`]}),` 159`,e.jsxs(i.li,{children:[e.jsx(i.a,{href:"/docs/architecture",children:"Architecture"}),` â Bastio + Ollama runs 160fully on-prem; the only off-host traffic is what you 161intentionally send to ClickHouse for analytics.`]}),` 162`]})]})}function d(s={}){const{wrapper:i}=s.components||{};return i?e.jsx(i,{...s,children:e.jsx(n,{...s})}):n(s)}export{d as default,a as extractedReferences,l as frontmatter,r as structuredData,h as toc};
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.