1"use strict";(globalThis.webpackChunktokligence_docs=globalThis.webpackChunktokligence_docs||[]).push([[967],{2614:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>t,contentTitle:()=>s,default:()=>h,frontMatter:()=>r,metadata:()=>l,toc:()=>c});const l=JSON.parse('{"id":"gateway/config/providers","title":"Provider Configuration","description":"Configure connections to LLM providers.","source":"@site/docs/gateway/config/providers.md","sourceDirName":"gateway/config","slug":"/gateway/config/providers","permalink":"/docs/gateway/config/providers","draft":false,"unlisted":false,"editUrl":"https://github.com/tokligence/tokligence-docs/tree/main/docs/gateway/config/providers.md","tags":[],"version":"current","sidebarPosition":1,"frontMatter":{"sidebar_position":1},"sidebar":"gatewaySidebar","previous":{"title":"Tool Calling","permalink":"/docs/gateway/features/tool-calling"},"next":{"title":"Environment Variables","permalink":"/docs/gateway/config/environment"}}');var a=i(4848),o=i(8453);const r={sidebar_position:1},s="Provider Configuration",t={},c=[{value:"Supported Providers",id:"supported-providers",level:2},{value:"OpenAI",id:"openai",level:3},{value:"Anthropic",id:"anthropic",level:3},{value:"Google Gemini",id:"google-gemini",level:3},{value:"Local LLMs",id:"local-llms",level:3},{value:"Ollama",id:"ollama",level:4},{value:"vLLM",id:"vllm",level:4},{value:"LM Studio",id:"lm-studio",level:4},{value:"Custom Endpoint",id:"custom-endpoint",level:3},{value:"Model Routing",id:"model-routing",level:2},{value:"Pattern Syntax",id:"pattern-syntax",level:3},{value:"Examples",id:"examples",level:3},{value:"Sidecar Translation",id:"sidecar-translation",level:2},{value:"Provider-Specific Settings",id:"provider-specific-settings",level:2},{value:"Rate Limits",id:"rate-limits",level:3},{value:"Timeouts",id:"timeouts",level:3},{value:"Prompt Caching",id:"prompt-caching",level:2},{value:"Health Checks",id:"health-checks",level:2}];function d(e){const n={code:"code",h1:"h1",h2:"h2",h3:"h3",h4:"h4",header:"header",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,o.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.header,{children:(0,a.jsx)(n.h1,{id:"provider-configuration",children:"Provider Configuration"})}),"\n",(0,a.jsx)(n.p,{children:"Configure connections to LLM providers."}),"\n",(0,a.jsx)(n.h2,{id:"supported-providers",children:"Supported Providers"}),"\n",(0,a.jsx)(n.h3,{id:"openai",children:"OpenAI"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"export TOKLIGENCE_OPENAI_API_KEY=sk-...\n"})}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.strong,{children:"Available Models:"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"gpt-4, gpt-4-turbo, gpt-4o"}),"\n",(0,a.jsx)(n.li,{children:"gpt-3.5-turbo"}),"\n",(0,a.jsx)(n.li,{children:"text-embedding-ada-002"}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"anthropic",children:"Anthropic"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"export TOKLIGENCE_ANTHROPIC_API_KEY=sk-ant-...\n"})}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.strong,{children:"Available Models:"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"claude-3-opus, claude-3-sonnet, claude-3-haiku"}),"\n",(0,a.jsx)(n.li,{children:"claude-3-5-sonnet, claude-3-5-haiku"}),"\n",(0,a.jsx)(n.li,{children:"claude-2.1, claude-2.0"}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"google-gemini",children:"Google Gemini"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"export TOKLIGENCE_GOOGLE_API_KEY=AIza...\n"})}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.strong,{children:"Available Models:"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"gemini-pro"}),"\n",(0,a.jsx)(n.li,{children:"gemini-pro-vision"}),"\n",(0,a.jsx)(n.li,{children:"gemini-ultra"}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"local-llms",children:"Local LLMs"}),"\n",(0,a.jsx)(n.h4,{id:"ollama",children:"Ollama"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"# Start Ollama\nollama serve\nollama run llama3\n\n# Configure gateway (auto-detected)\n# No API key needed\n"})}),"\n",(0,a.jsx)(n.h4,{id:"vllm",children:"vLLM"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"export TOKLIGENCE_LLM_ENDPOINT=http://localhost:8000/v1\n"})}),"\n",(0,a.jsx)(n.h4,{id:"lm-studio",children:"LM Studio"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"export TOKLIGENCE_LLM_ENDPOINT=http://localhost:1234/v1\n"})}),"\n",(0,a.jsx)(n.h3,{id:"custom-endpoint",children:"Custom Endpoint"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"export TOKLIGENCE_LLM_ENDPOINT=http://your-llm.com/v1\nexport TOKLIGENCE_LLM_API_KEY=optional-key\n"})}),"\n",(0,a.jsx)(n.h2,{id:"model-routing",children:"Model Routing"}),"\n",(0,a.jsx)(n.p,{children:"Configure model-to-provider routing:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'export TOKLIGENCE_ROUTES="claude*=>anthropic,gpt-*=>openai,gemini-*=>google"\n'})}),"\n",(0,a.jsx)(n.h3,{id:"pattern-syntax",children:"Pattern Syntax"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"*"})," matches any characters"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"=>"})," separates pattern from provider"]}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"examples",children:"Examples"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'# Default routing\nTOKLIGENCE_ROUTES="claude*=>anthropic,gpt-*=>openai,o1*=>openai,gemini-*=>google"\n\n# Custom vendor prefixes\nTOKLIGENCE_ROUTES="qwen*=>ali,llama*=>local"\n'})}),"\n",(0,a.jsx)(n.h2,{id:"sidecar-translation",children:"Sidecar Translation"}),"\n",(0,a.jsx)(n.p,{children:"For Claude Code \u2192 OpenAI translation:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'# Model mapping\nexport TOKLIGENCE_SIDECAR_MODEL_MAP="claude-3-5-sonnet-20241022=gpt-4o\nclaude-3-5-h
1aiku-20241022=gpt-4o-mini"\n\n# Default model for unmapped requests\nexport TOKLIGENCE_SIDECAR_DEFAULT_OPENAI_MODEL=gpt-4o\n\n# Max tokens (Anthropic allows larger than OpenAI)\nexport TOKLIGENCE_OPENAI_COMPLETION_MAX_TOKENS=16384\n'})}),"\n",(0,a.jsx)(n.h2,{id:"provider-specific-settings",children:"Provider-Specific Settings"}),"\n",(0,a.jsx)(n.h3,{id:"rate-limits",children:"Rate Limits"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-ini",children:"# config/gateway.ini\n[openai]\nrate_limit_rpm = 100\n\n[anthropic]\nrate_limit_rpm = 50\n"})}),"\n",(0,a.jsx)(n.h3,{id:"timeouts",children:"Timeouts"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"export TOKLIGENCE_REQUEST_TIMEOUT=60000 # milliseconds\n"})}),"\n",(0,a.jsx)(n.h2,{id:"prompt-caching",children:"Prompt Caching"}),"\n",(0,a.jsx)(n.p,{children:"Enable Anthropic's prompt caching for cost optimization:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-json",children:'{\n "model": "claude-3-5-sonnet-20241022",\n "messages": [...],\n "cache_control": {\n "type": "ephemeral"\n }\n}\n'})}),"\n",(0,a.jsx)(n.h2,{id:"health-checks",children:"Health Checks"}),"\n",(0,a.jsx)(n.p,{children:"Verify provider connectivity:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"curl http://localhost:8081/health\n"})}),"\n",(0,a.jsx)(n.p,{children:"Response:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-json",children:'{\n "status": "ok",\n "providers": {\n "openai": "connected",\n "anthropic": "connected",\n "google": "connected"\n }\n}\n'})})]})}function h(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(d,{...e})}):d(e)}},8453:(e,n,i)=>{i.d(n,{R:()=>r,x:()=>s});var l=i(6540);const a={},o=l.createContext(a);function r(e){const n=l.useContext(o);return l.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function s(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:r(e.components),l.createElement(o.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.