1"use strict";(self.webpackChunkmy_website=self.webpackChunkmy_website||[]).push([[6138],{4418:(e,n,s)=>{s.r(n),s.d(n,{assets:()=>r,contentTitle:()=>a,default:()=>h,frontMatter:()=>l,metadata:()=>i,toc:()=>d});var t=s(4848),o=s(8453);const l={},a=void 0,i={id:"ai-assisted-coding/local-models-slop-mastery",title:"local-models-slop-mastery",description:"Local LLMs basics",source:"@site/docs/ai-assisted-coding/04-local-models-slop-mastery.md",sourceDirName:"ai-assisted-coding",slug:"/ai-assisted-coding/local-models-slop-mastery",permalink:"/docs/ai-assisted-coding/local-models-slop-mastery",draft:!1,unlisted:!1,tags:[],version:"current",sidebarPosition:4,frontMatter:{},sidebar:"tutorialSidebar",previous:{title:"cursor-slop-mastery",permalink:"/docs/ai-assisted-coding/cursor-slop-mastery"},next:{title:"agentic-AI-development",permalink:"/docs/ai-assisted-coding/agentic-AI-development"}},r={},d=[{value:"Local LLMs basics",id:"local-llms-basics",level:2},{value:"Theory",id:"theory",level:3},{value:"Quantization",id:"quantization",level:4},{value:"Offloading",id:"offloading",level:4},{value:"Lm studio",id:"lm-studio",level:2},{value:"CLI",id:"cli",level:4},{value:"Programming",id:"programming",level:4},{value:"Open AI Compatibility",id:"open-ai-compatibility",level:5},{value:"LM studio TS sdk",id:"lm-studio-ts-sdk",level:5},{value:"OLlama",id:"ollama",level:2},{value:"rUnning models",id:"running-models",level:4},{value:"managing models",id:"managing-models",level:4},{value:"Model env vars",id:"model-env-vars",level:4},{value:"Modelfiles",id:"modelfiles",level:4},{value:"Ollama server",id:"ollama-server",level:4},{value:"Ollama API",id:"ollama-api",level:4},{value:"Important local models",id:"important-local-models",level:2},{value:"OpenAI Whisper",id:"openai-whisper",level:3},{value:"Python",id:"python",level:4},{value:"Command line",id:"command-line",level:4},{value:"TranslateGemma",id:"translategemma",level:3}];function c(e){const n={a:"a",code:"code",div:"div",h2:"h2",h3:"h3",h4:"h4",h5:"h5",img:"img",li:"li",ol:"ol",p:"p",path:"path",pre:"pre",strong:"strong",svg:"svg",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,o.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(n.h2,{id:"local-llms-basics",children:"Local LLMs basics"}),"\n",(0,t.jsx)(n.p,{children:"You can use local LLMs in a chat interface either with LMStudio desktop app or the Ollama CLI."}),"\n",(0,t.jsx)(n.p,{children:"You can download quantized models off of hugging face or in LM Studio itself."}),"\n",(0,t.jsx)(n.h3,{id:"theory",children:"Theory"}),"\n",(0,t.jsx)(n.h4,{id:"quantization",children:"Quantization"}),"\n",(0,t.jsxs)(n.p,{children:[(0,t.jsx)(n.strong,{children:"quantization"})," is the idea of precision in model parameters, either letting each parameter have a floating point precision (more precise) or an integer precision (less precise)."]}),"\n",(0,t.jsx)(n.p,{children:"Although it sounds like being more precise would lead to better results - which it does - it also adds up more space to download local models and requires more RAM. To use a model for inference, it has to get loaded into memory, and even the smallest LLM has over 1 billion parameters. Higher precision leads to higher RAM requirements:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"A model with float32 quantization for parameters means each parameter is 32 bits, or 4 bytes, meaning a model with 2 billion parameters would need 8GB of RAM."}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:"Thus quantization allows us to mathematically round the floating point precision parameters to integers, either int4 (4 bit) or int8 (8 bit) to cut down the RAM usage of a model:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"A model quantized with int4 quantization for parameters means each parameter is 4 bits, or 0.5 bytes, meaning a model with 2 billion parameters would only need 1GB of RAM."}),"\n"]}),"\n",(0,t.jsxs)(n.div,{className:"markdown-alert markdown-alert-tip",dir:"auto",children:["\n",(0,t.jsxs)(n.p,{className:"markdown-alert-title",dir:"auto",children:[(0,t.jsx)(n.svg,{className:"octicon",viewBox:"0 0 16 16",width:"16",height:"16","aria-hidden":"true",children:(0,t.jsx)(n.path,{d:"M8 1.5c-2.363 0-4 1.69-4 3.75 0 .984.424 1.625.984 2.304l.214.253c.223.264.47.556.673.848.284.411.537.896.621 1.49a.75.75 0 0 1-1.484.211c-.04-.282-.163-.547-.37-.847a8.456 8.456 0 0 0-.542-.68c-.084-.1-.173-.205-.268-.32C3.201 7.75 2.5 6.766 2.5 5.25 2.5 2.31 4.863 0 8 0s5.5 2.31 5.5 5.25c0 1.516-.701 2.5-1.328 3.259-.095.115-.184.22-.268.319-.207.245-.383.453-.541.681-.208.3-.33.565-.37.847a.751.751 0 0 1-1.485-.212c.084-.593.337-1.078.621-1.489.203-.292.45-.584.673-.848.075-.088.147-.173.213-.253.561-.679.985-1.32.985-2.304 0-2.06-1.637-3.75-4-3.75ZM5.75 12h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1 0-1.5ZM6 15.25a.75.75 0 0 1 .75-.75h2.5a.75.75 0 0 1 0 1.5h-2.5a.75.75 0 0 1-.75-.75Z"})}),"TIP"]}),"\n",(0,t.jsx)(n.p,{children:"quantization allows us to achieve up to 1/2 or 1/4 cutting of RAM usage, while still having only a negligible difference in performance from the more precise unquantized models."}),"\n"]}),"\n",(0,t.jsx)(n.h4,{id:"offloading",children:"Offloading"}),"\n",(0,t.jsxs)(n.p,{children:[(0,t.jsx)(n.strong,{children:"offloading"})," is the technique of loading a model's parameters between CPU, GPU, and RAM, in order to efficie
1ntly load a model in memory."]}),"\n",(0,t.jsx)(n.p,{children:"A main drawback of offloading is that model performance becomes worse, even if memoyr use is more efficient."}),"\n",(0,t.jsx)(n.h2,{id:"lm-studio",children:"Lm studio"}),"\n",(0,t.jsx)(n.h4,{id:"cli",children:"CLI"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"lms ls"}),": lists all downloaded models"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"lms ps"}),": lists all currently loaded models in memory"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"lms load <model-id>"}),": loads a specific model"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"lms unload <model-id>"}),": unloads a specific model"]}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"listing models"})}),"\n",(0,t.jsxs)(n.p,{children:["Show all downloaded models using the ",(0,t.jsx)(n.code,{children:"lms ls"})," command. You have 4 options to consider:"]}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"--llm"}),": lists only llm models"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"--json"}),": lists info in JSON"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"--detailed"}),": lists details info"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"--embeddings"}),": prints only embedding models"]}),"\n"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms ls\n"})}),"\n",(0,t.jsx)(n.p,{children:"Example output:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"You have 47 models, taking up 160.78 GB of disk space.\n\nLLMs (Large Language Models) PARAMS ARCHITECTURE SIZE\nlmstudio-community/meta-llama-3.1-8b-instruct 8B Llama 4.92 GB\nhugging-quants/llama-3.2-1b-instruct 1B Llama 1.32 GB\nmistral-7b-instruct-v0.3 Mistral 4.08 GB\nzeta 7B Qwen2 4.09 GB\n\n... (abbreviated in this example) ...\n\nEmbedding Models PARAMS ARCHITECTURE SIZE\ntext-embedding-nomic-embed-text-v1.5@q4_k_m Nomic BERT 84.11 MB\ntext-embedding-bge-small-en-v1.5 33M BERT 24.81 MB\n"})}),"\n",(0,t.jsx)(n.p,{children:"List only LLM models:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms ls --llm\n"})}),"\n",(0,t.jsx)(n.p,{children:"List only embedding models:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms ls --embedding\n"})}),"\n",(0,t.jsx)(n.p,{children:"Get detailed information about models:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms ls --detailed\n"})}),"\n",(0,t.jsx)(n.p,{children:"Output in JSON format:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms ls --json\n"})}),"\n",(0,t.jsxs)(n.p,{children:["You can show all currently loaded models with ",(0,t.jsx)(n.code,{children:"lms ps"}),"."]}),"\n",(0,t.jsx)(n.p,{children:"Get the list in machine-readable format:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms ps --json\n"})}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"loading into memory"})}),"\n",(0,t.jsx)(n.p,{children:"Load a model into memory by running the following command:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms load <model_key>\n"})}),"\n",(0,t.jsxs)(n.p,{children:["You can find the\xa0",(0,t.jsx)(n.code,{children:"model_key"}),"\xa0by first running\xa0",(0,t.jsx)(n.a,{href:"https://lmstudio.ai/docs/cli/ls",children:(0,t.jsx)(n.code,{children:"lms ls"})}),"\xa0to list your locally downloaded models. You also have access to these options:"]}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.img,{src:"https://i.imgur.com/v3faHML.jpeg",alt:""})}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"unloading from memory"})}),"\n",(0,t.jsx)(n.p,{children:"Unload a single model from memory by running:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms unload <model_key>\n"})}),"\n",(0,t.jsx)(n.p,{children:"If no model key is provided, you will be prompted to select from currently loaded models."}),"\n",(0,t.jsx)(n.p,{children:"To unload all currently loaded models at once:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms unload --all\n"})}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"Server CLI"})}),"\n",(0,t.jsxs)(n.p,{children:["You use the ",(0,t.jsx)(n.code,{children:"lms server start"})," command to start the LM studio server"]}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"lms server start"}),": starts server with default settings on port 1234"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"lms server start --port <port>"}),": starts server on specific port"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"lms server start --cors"}),": opens CORS for all web apps to access"]}),"\n"]}),"\n",(0,t.jsxs)(n.p,{children:["You can use the ",(0,t.jsx)(n.code,{children:"lms server stop"})," command to stop the LM studio server."]}),"\n",(0,t.jsxs)(n.p,{children:["You use the ",(0,t.jsx)(n.code,{children:"lms server status"})," command to see the status of the LM studio server"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-bash",children:"lms server start # start server\nlms server status # get status\nlms server stop # stop server\n"})}),"\n",(0,t.jsx)(n.p,{children:"You also have these options:"}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.img,{src:"https://i.imgur.com/EcHQo7l.png",alt:""})}),"\n",(0,t.jsx)(n.p,{children:"Get the status in machine-readable JSON format:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms server status --json --quiet\n"})}),"\n",(0,t.jsx)(n.p,{children:"Example output:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-json",children:'{"running":true,"port":1234}\n'})}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"seeing logs"})}),"\n",(0,t.jsxs)(n.p,{children:[(0,t.jsx)(n.code,{children:"lms log stream"}),"\xa0allows you to inspect the exact input string that goes to the model."]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"lms log stream\n"})}),"\n",(0,t.jsx)(n.p,{children:"Here would be the example output:"}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.img,{src:"https://i.imgur.com/yr5QsPH.jpeg",alt:""})}),"\n",(0,t.jsx)(n.h4,{id:"programming",children:"Programming"}),"\n",(0,t.jsxs)(n.p,{children:["You can hit API endpoints for models that your load onto the LM studio server, which runs on ",(0,t.jsx)(n.code,{children:"localhost:1234"}),"."]}),"\n",(0,t.jsx)(n.p,{children:"There are three different ways to run the LMS studio server and hit up the endpoints:"}),"\n",(0,t.jsxs)(n.ol,{children:["\n",(0,t.jsx)(n.li,{children:"Basic rest API"}),"\n",(0,t.jsx)(n.li,{children:"Open AI SDK (compatibility version)"}),"\n",(0,t.jsx)(n.li,{children:"LM studio Python SDK"}),"\n",(0,t.jsx)(n.li,{children:"LM studio TS SDK"}),"\n"]}),"\n",(0,t.jsx)(n.h5,{id:"open-ai-compatibility",children:"Open AI Compatibility"}),"\n",(0,t.jsxs)(n.p,{children:["Using models with LM studio is completely compatible with the openAI sdk. All you have to do is to pass the ",(0,t.jsx)(n.code,{children:"base_url"})," parameter and point that to the LM studio server endpoint, like so:"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-python",children:'from openai import OpenAI\n\nclient = OpenAI(\n base_url="http://localhost:1234/v1", # LM Studio endpoint on port 1234\n api_key="something-doesnt-matter", # doesn\'t matter, but should pass value\
1n)\n'})}),"\n",(0,t.jsx)(n.p,{children:"And here is an example showing just how simple and compatible the OpenAI SDK is to use with LM studio models"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-python",children:'from openai import OpenAI\n\nclient = OpenAI(\n base_url="http://localhost:1234/v1",\n api_key="something-doesnt-matter",\n)\n\nresponse = client.chat.completions.create(\n model="gemma-3-12b-it-qat",\n messages=[\n {\n "role": "system",\n "content": "You are a helpful and friendly assistant."\n },\n {\n "role": "user",\n "content": "What is the meaning of life?"\n }\n ],\n temperature=0.7,\n)\n\nprint(response.choices[0].message.content)\n'})}),"\n",(0,t.jsx)(n.h5,{id:"lm-studio-ts-sdk",children:"LM studio TS sdk"}),"\n",(0,t.jsx)(n.p,{children:"fIrst install with this:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-bash",children:"npm install @lmstudio/sdk --save\n"})}),"\n",(0,t.jsx)(n.p,{children:"And here's a quickstart:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-ts",children:'import { LMStudioClient } from "@lmstudio/sdk";\nconst client = new LMStudioClient();\n\nconst model = await client.llm.model("llama-3.2-1b-instruct");\nconst result = await model.respond("What is the meaning of life?");\n\nconsole.info(result.content);\n'})}),"\n",(0,t.jsx)(n.h2,{id:"ollama",children:"OLlama"}),"\n",(0,t.jsx)(n.p,{children:"OLLama is a CLI tool for installing and running local models. Here is an example that automatically installs and runs llama 3.2"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-bash",children:"ollama run llama3.2\n"})}),"\n",(0,t.jsx)(n.p,{children:"In fact, here's a list of all CLI commands you can run:"}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.img,{src:"https://i.imgur.com/acwfO9j.jpeg",alt:""})}),"\n",(0,t.jsx)(n.h4,{id:"running-models",children:"rUnning models"}),"\n",(0,t.jsx)(n.p,{children:"When chatting with ollama models, you have access to these slash commands:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"Available Commands:\n /set Set session variables\n /show Show model information\n /load <model> Load a session or model\n /save <model> Save your current session\n /clear Clear session context\n /bye Exit\n /?, /help Help for a command\n /? shortcuts Help for keyboard shortcuts\n"})}),"\n",(0,t.jsx)(n.p,{children:"Since you have to chat using the CLI in a purely text based ways, there are a few caveats to keep iin mind when trying to chat with OLLama:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.strong,{children:"multiline text"}),": ANy multiline text needs to be encased in triple double quotes"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.strong,{children:"images"}),": To refer to images or files, you just write out the relative path to that file in your prompt. Any filepaths you refer to MUST MUST MUST be at the end of your prompt, after any text."]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.strong,{children:"system message"}),": run the ",(0,t.jsx)(n.code,{children:"/set system <message>"})," command to change the model's system message for the chat duration"]}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"saving chats"})}),"\n",(0,t.jsxs)(n.p,{children:["To save chats, you can use the ",(0,t.jsx)(n.code,{children:"/save <chat-name>"})," and ",(0,t.jsx)(n.code,{children:"/load <chat-name>"})," to load a chat. These commands save and load the chat respectively with the hyperparameters, chat history, and system message all set and saved."]}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"/show command"})}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"Available Commands:\n /show info Show details for this model\n /show license Show model license\n /show modelfile Show Modelfile for this model\n /show parameters Show parameters for this model\n /show system Show system message\n /show template Show prompt template\n"})}),"\n",(0,t.jsxs)(n.p,{children:["If you run the ",(0,t.jsx)(n.code,{children:"/show system"})," command, you can see the system message for the model."]}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"/set command"})}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:">>> /set\nAvailable Commands:\n /set parameter ... Set a parameter\n /set system <string> Set system message\n /set history Enable history\n /set nohistory Disable history\n /set wordwrap Enable wordwrap\n /set nowordwrap Disable wordwrap\n /set format json Enable JSON mode\n /set noformat Disable formatting\n /set verbose Show LLM stats\n /set quiet Disable LLM stats\n /set think Enable thinking\n /set nothink Disable thinking\n"})}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"/set system <message>"})," : changes the model's system message for the chat duration"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"/set parameter"}),": shows the parameters of the model you can change"]}),"\n"]}),"\n",(0,t.jsx)(n.h4,{id:"managing-models",children:"managing models"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"ollama list"}),": lists all models you have installed."]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"ollama ps"}),": lists all currently running models."]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"ollama rm <model-name>"}),": deletes a model by its name."]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"ollama show <model-name>"}),": shows more info on the specified model."]}),"\n"]}),"\n",(0,t.jsxs)(n.div,{className:"markdown-alert markdown-alert-tip",dir:"auto",children:["\n",(0,t.jsxs)(n.p,{className:"markdown-alert-title",dir:"auto",children:[(0,t.jsx)(n.svg,{className:"octicon",viewBox:"0 0 16 16",width:"16",height:"16","aria-hidden":"true",children:(0,t.jsx)(n.path,{d:"M8 1.5c-2.363 0-4 1.69-4 3.75 0 .984.424 1.625.984 2.304l.214.253c.223.264.47.556.673.848.284.411.537.896.621 1.49a.75.75 0 0 1-1.484.211c-.04-.282-.163-.547-.37-.847a8.456 8.456 0 0 0-.542-.68c-.084-.1-.173-.205-.268-.32C3.201 7.75 2.5 6.766 2.5 5.25 2.5 2.31 4.863 0 8 0s5.5 2.31 5.5 5.25c0 1.516-.701 2.5-1.328 3.259-.095.115-.184.22-.268.319-.207.245-.383.453-.541.681-.208.3-.33.565-.37.847a.751.751 0 0 1-1.485-.212c.084-.593.337-1.078.621-1.489.203-.292.45-.584.673-.848.075-.088.147-.173.213-.253.561-.679.985-1.32.985-2.304 0-2.06-1.637-3.75-4-3.75ZM5.75 12h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1 0-1.5ZM6 15.25a.75.75 0 0 1 .75-.75h2.5a.75.75 0 0 1 0 1.5h-2.5a.75.75 0 0 1-.75-.75Z"})}),"TIP"]}),"\n",(0,t.jsxs)(n.p,{children:["You can find the parameters for a model on the ollama page for a model or through ",(0,t.jsx)(n.code,{children:"ollama show"})," command."]}),"\n"]}),"\n",(0,t.jsx)(n.h4,{id:"model-env-vars",children:"Model env vars"}),"\n",(0,t.jsx)(n.p,{children:"These are the env vars you should set on a model in order to make OLLAMA more efficient:"}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.img,{src:"https://i.imgur.com/aq94hHQ.jpeg",alt:""})}),"\n",(0,t.jsx)(n.h4,{id:"modelfiles",children:"Modelfiles"}),"\n",(0,t.jsxs)(n.p,{children:[(0,t.jsx)(n.strong,{children:"Modelfiles"})," are essentially the Dockerfile version of creating LLMs, blueprinting them with system prompts, hyperparameter values, and message history."]}),"\n",(0,t.jsx)(n.p,{children:"Here are the directives you can use:"}),"\n",(0,t.jsxs)(n.table,{children:[(0,t.jsx)(n.thead,{children:(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.th,{children:"Instruction"}),(0,t.jsx)(n.th,{children:"Description"})]})}),(0,t.jsxs)(n.tbody,{children:[(0,t.jsxs)(n.tr,{children:[(0,t.jsxs)(n.td,{children:[(0,t.jsx)(n.a,{href:"https://github.com/ollama/ollama/blob/main/docs/modelfile.md#from-required",children:(0,t.jsx)(n.code,{children:"FROM"})})," (required)"]}),(0,t.jsx)(n.td,{children:"Defines the base model to use."})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.a,{href:"https://github.com/ollama/ollama/blob/main/docs/modelfile.md#parameter",children:(0,t.jsx)(n.code,{children:"PARAMETER"})})}),(0,t.jsx)(n.td,{children:"Sets the parameters for how Ollama will run the model."})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.a,{href:"https://github.com/ollama/ollama/blob/main/docs/modelfile.md#template",children:(0,t.jsx)(n.code,{children:"TEMPLATE"})})}),(0,t.jsx)(n.td,{children:"The full prompt template to be sent to the model."})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.a,{href:"https://github.com/ollama/ollama/blob/main/docs/modelfile.md#system",children:(0,t.jsx)(n.code,{children:"SYSTEM"})})}),(0,t.jsx)(n.td,{children:"Specifies the system message that will be set in the template."})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.a,{href:"https://github.com/ollama/ollama/blob/main/docs/modelfile.md#adapter",children:(0,t.jsx)(n.code,{children:"ADAPTER"})})}),(0,t.jsx)(n.td,{children:"Defines the (Q)LoRA adapters to apply to the model."})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.a,{href:"https://github.com/ollama/ollama/blob/main/docs/modelfile.md#license",children:(0,t.jsx)(n.code,{children:"LICENSE"})})}),(0,t.jsx)(n.td,{children:"Specifies the legal license."})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.a,{href:"https://github.com/ollama/ollama/blob/main/docs/modelfile.md#message",children:(0,t.jsx)(n.code,{children:"MESSAGE"})})}),(0,t.jsx)(n.td,{children:"Specify message history."})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:"Here is an example modelfile:"}),(0,t.jsx)(n.td,{})]})]})]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-dockerfile",children:'FROM llama3.2\n# sets the temperature to 1 [higher is more creative, lower is more coherent]\nPARAMETER temperature 1\n# set
1s the context window size to 4096, this controls how many tokens the LLM can use as context to generate the next token\nPARAMETER num_ctx 4096\n\n# sets a custom system message to specify the behavior of the chat assistant\nSYSTEM You are Mario from super mario bros, acting as an assistant.\n\n# adds message history\nMESSAGE user "Hi mario, what\'s up?"\nMESSAGE assistant "whats a up mamma mia you piece of shit!"\n'})}),"\n",(0,t.jsx)(n.p,{children:"To use this:"}),"\n",(0,t.jsxs)(n.ol,{children:["\n",(0,t.jsxs)(n.li,{children:["Save it as a file (e.g.\xa0",(0,t.jsx)(n.code,{children:"Modelfile"}),")"]}),"\n",(0,t.jsx)(n.li,{children:(0,t.jsx)(n.code,{children:"ollama create choose-a-model-name -f <location of the file e.g. ./Modelfile>"})}),"\n",(0,t.jsx)(n.li,{children:(0,t.jsx)(n.code,{children:"ollama run choose-a-model-name"})}),"\n",(0,t.jsx)(n.li,{children:"Start using the model!"}),"\n"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-bash",children:"# 1. create the modelfile and use it\nollama create <new-model-name> -f ./Modelfile\nollama run <new-model-name>\n"})}),"\n",(0,t.jsxs)(n.p,{children:["To view the Modelfile of a given model, use the\xa0",(0,t.jsx)(n.code,{children:"ollama show --modelfile"}),"\xa0command."]}),"\n",(0,t.jsx)(n.h4,{id:"ollama-server",children:"Ollama server"}),"\n",(0,t.jsxs)(n.p,{children:["Run ",(0,t.jsx)(n.code,{children:"ollama serve"})," to start the server, but ollama runs on ",(0,t.jsx)(n.code,{children:"localhost:11434"})," automatically when you start it."]}),"\n",(0,t.jsx)(n.h4,{id:"ollama-api",children:"Ollama API"}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"API fetching"})}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"open ai compatible"})}),"\n",(0,t.jsxs)(n.p,{children:["You can use the openAI compatibility API through setting the ",(0,t.jsx)(n.code,{children:"baseUrl"})," property to ",(0,t.jsx)(n.code,{children:"localhost:11434/v1"})," endpoint."]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{})}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"vercel ai"})}),"\n",(0,t.jsx)(n.p,{children:"Through the openAI compatibility endpoint, you can use ollama models on vercel AI."}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-ts",children:'import { createOpenAICompatible } from "npm:@ai-sdk/openai-compatible";\n\n function get_ollama(modelName: string) {\n const model = createOpenAICompatible({\n name: "ollama",\n baseURL: `http://localhost:11434/v1`,\n apiKey: "1234567890",\n });\n return {\n model: model(modelName),\n modelOptions: {\n maxRetries: 0,\n },\n };\n },\n'})}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"python sdk"})}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"js sdk"})}),"\n",(0,t.jsxs)(n.p,{children:["The JS sdk is super easy to use through the ",(0,t.jsx)(n.code,{children:"ollama"})," package:"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-ts",children:"import ollama from 'ollama'\n\nconst response = await ollama.chat({\n model: 'llama3.1',\n messages: [{ role: 'user', content: 'Why is the sky blue?' }],\n})\nconsole.log(response.message.content)\n"})}),"\n",(0,t.jsx)(n.p,{children:"You can also stream messages:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-ts",children:"import ollama from 'ollama'\n\nconst message = { role: 'user', content: 'Why is the sky blue?' }\nconst response = await ollama.chat({\n model: 'llama3.1',\n messages: [message],\n stream: true,\n})\nfor await (const part of response) {\n process.stdout.write(part.message.content)\n}\n"})}),"\n",(0,t.jsx)(n.p,{children:(0,t.jsx)(n.strong,{children:"custom sdk"})}),"\n",(0,t.jsx)(n.p,{children:"Made with the power of gemini"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-ts",children:'import { z } from "npm:zod";\n\n// Base URL for the Ollama API\nconst OLLAMA_API_BASE_URL = "http://localhost:11434/api";\n\n// Zod Schemas for API validation\n\nconst ModelDetailsSchema = z.object({\n parent_model: z.string(),\n format: z.string(),\n family: z.string(),\n families: z.array(z.string()).nullable(),\n parameter_size: z.string(),\n quantization_level: z.string(),\n});\n\nconst ModelSchema = z.object({\n name: z.string(),\n model: z.string(),\n modified_at: z.string(),\n size: z.number(),\n digest: z.string(),\n details: ModelDetailsSchema,\n});\n\nconst ListModelsResponseSchema = z.object({\n models: z.array(ModelSchema),\n});\n\nconst GenerateCompletionOptionsSchema = z\n .object({\n temperature: z.number().optional(),\n seed: z.number().optional(),\n top_k: z.number().optional(),\n top_p: z.number().optional(),\n min_p: z.number().optional(),\n repeat_last_n: z.number().optional(),\n repeat_penalty: z.number().optional(),\n presence_penalty: z.number().optional(),\n frequency_penalty: z.number().optional(),\n stop: z.array(z.string()).optional(),\n })\n .partial();\n\nconst GenerateCompletionRequestSchema = z.object({\n model: z.string(),\n prompt: z.string(),\n suffix: z.string().optional(),\n images: z.array(z.string()).optional(),\n think: z.boolean().optional(),\n format: z.union([z.literal("json"), z.any()]).optional(),\n options: GenerateCompletionOptionsSchema.optional(),\n stream: z.boolean().optional(),\n raw: z.boolean().optional(),\n keep_alive: z.string().optional(),\n});\n\nconst GenerateCompletionResponseSchema = z.object({\n model: z.string(),\n created_at: z.string(),\n response: z.string(),\n done: z.boolean(),\n context: z.array(z.number()).optional(),\n total_duration: z.number().optional(),\n load_duration: z.number().optional(),\n prompt_eval_count: z.number().optional(),\n prompt_eval_duration: z.number().optional(),\n eval_count: z.number().optional(),\n eval_duration: z.number().optional(),\n});\n\nconst MessageSchema = z.object({\n role: z.enum(["system", "user", "assistant", "tool"]),\n content: z.string(),\n images: z.array(z.string()).optional(),\n});\n\nconst GenerateChatRequestSchema = z.object({\n model: z.string(),\n messages: z.array(MessageSchema),\n tools: z.array(z.any()).optional(),\n think: z.boolean().optional(),\n format: z.union([z.literal("json"), z.any()]).optional(),\n options: GenerateCompletionOptionsSchema.optional(),\n stream: z.boolean().optional(),\n keep_alive: z.string().optional(),\n});\n\nconst GenerateChatResponseSchema = z.object({\n model: z.string(),\n created_at: z.string(),\n message: MessageSchema,\n done: z.boolean(),\n total_duration: z.number().optional(),\n load_duration: z.number().optional(),\n prompt_eval_count: z.number().optional(),\n prompt_eval_duration: z.number().optional(),\n eval_count: z.number().optional(),\n eval_duration: z.number().optional(),\n});\n\nconst GenerateEmbeddingsRequestSchema = z.object({\n model: z.string(),\n input: z.union([z.string(), z.array(z.string())]),\n truncate: z.boolean().optional(),\n options: GenerateCompletionOptionsSchema.optional(),\n keep_alive: z.string().optional(),\n});\n\nconst GenerateEmbeddingsResponseSchema = z.object({\n model: z.string(),\n embeddings: z.array(z.array(z.number())),\n total_duration: z.number().optional(),\n load_duration: z.number().optional(),\n prompt_eval_count: z.number().optional(),\n});\n\n// Type Definitions from Zod Schemas\ntype ListModelsResponse = z.infer<typeof ListModelsResponseSchema>;\ntype GenerateCompletionRequest = z.infer<\n typeof GenerateCompletionRequestSchema\n>;\ntype GenerateCompletionResponse = z.infer<\n typeof GenerateCompletionResponseSchema\n>;\ntype GenerateChatRequest = z.infer<typeof GenerateChatRequestSchema>;\ntype GenerateChatResponse = z.infer<typeof GenerateChatResponseSchema>;\ntype GenerateEmbeddingsRequest = z.infer<\n typeof GenerateEmbeddingsRequestSchema\n>;\ntype GenerateEmbeddingsResponse = z.infer<\n typeof GenerateEmbeddingsResponseSchema\n>;\n\n/**\n * A TypeScript class to interact with the Ollama API in a typesafe way.\n */\nexport class OllamaAPI {\n private baseUrl: string;
1\n\n constructor(baseUrl: string = OLLAMA_API_BASE_URL) {\n this.baseUrl = baseUrl;\n }\n\n private async post<T>(endpoint: string, body: unknown): Promise<T> {\n const response = await fetch(`${this.baseUrl}${endpoint}`, {\n method: "POST",\n headers: {\n "Content-Type": "application/json",\n },\n body: JSON.stringify(body),\n });\n\n if (!response.ok) {\n throw new Error(`API request failed with status ${response.status}`);\n }\n\n return response.json();\n }\n\n private async get<T>(endpoint: string): Promise<T> {\n const response = await fetch(`${this.baseUrl}${endpoint}`);\n if (!response.ok) {\n throw new Error(`API request failed with status ${response.status}`);\n }\n return response.json();\n }\n\n /**\n * Lists all models available locally.\n */\n async listModels(): Promise<ListModelsResponse> {\n const response = await this.get("/tags");\n return ListModelsResponseSchema.parse(response);\n }\n\n /**\n * Generates a completion for a given prompt.\n * @param request The request object for generating a completion.\n * @returns The generated completion.\n */\n async generateCompletion(\n request: GenerateCompletionRequest\n ): Promise<GenerateCompletionResponse> {\n const validatedRequest = GenerateCompletionRequestSchema.parse(request);\n const response = await this.post("/generate", validatedRequest);\n return GenerateCompletionResponseSchema.parse(response);\n }\n\n /**\n * Generates the next message in a chat.\n * @param request The request object for generating a chat completion.\n * @returns The generated chat message.\n */\n async generateChat(\n request: GenerateChatRequest\n ): Promise<GenerateChatResponse> {\n const validatedRequest = GenerateChatRequestSchema.parse(request);\n const response = await this.post("/chat", validatedRequest);\n return GenerateChatResponseSchema.parse(response);\n }\n\n /**\n * Generates embeddings for a given input.\n * @param request The request object for generating embeddings.\n * @returns The generated embeddings.\n */\n async generateEmbeddings(\n request: GenerateEmbeddingsRequest\n ): Promise<GenerateEmbeddingsResponse> {\n const validatedRequest = GenerateEmbeddingsRequestSchema.parse(request);\n const response = await this.post("/embed", validatedRequest);\n return GenerateEmbeddingsResponseSchema.parse(response);\n }\n}\n\n'})}),"\n",(0,t.jsx)(n.h2,{id:"important-local-models",children:"Important local models"}),"\n",(0,t.jsx)(n.h3,{id:"openai-whisper",children:"OpenAI Whisper"}),"\n",(0,t.jsx)(n.h4,{id:"python",children:"Python"}),"\n",(0,t.jsx)(n.p,{children:"Here is how you can use open ai whisper to transcribe or translate audio files:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-ts",children:'import whisper\n\nmodel = whisper.load_model("base.en")\n\nfilepath = os.path.join(pathlib.Path.home(), "Downloads", "totranscribe.webm")\nresult = model.transcribe(filepath)\n\nprint(result["text"])\n'})}),"\n",(0,t.jsx)(n.p,{children:"Here are the different models you have access to, all unquantized."}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:'"tiny.en"'}),": the smallest english version, at 39M params"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:'"base.en"'}),": the smallest english version, at 74M params"]}),"\n"]}),"\n",(0,t.jsx)(n.h4,{id:"command-line",children:"Command line"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"whisper <audio-file-path>"}),": transcribes the audio file with auto detecting the language"]}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:"Here are the different options you have"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"--model <model>"}),": chooses the specific model"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.code,{children:"--language <language-code>"}),": specifies the language the audio file is in. Pass in a language code, like ",(0,t.jsx)(n.code,{children:"en"})," or ",(0,t.jsx)(n.code,{children:"es"}),"."]}),"\n"]}),"\n",(0,t.jsx)(n.h3,{id:"translategemma",children:"TranslateGemma"}),"\n",(0,t.jsx)(n.p,{children:"This is an Ollama LLM that excels at translation."})]})}function h(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,t.jsx)(n,{...e,children:(0,t.jsx)(c,{...e})}):c(e)}},8453:(e,n,s)=>{s.d(n,{R:()=>a,x:()=>i});var t=s(6540);const o={},l=t.createContext(o);function a(e){const n=t.useContext(l);return t.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function i(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:a(e.components),t.createElement(l.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.