https://robocloud-dashboard.vercel.app/assets/VisionLanguageModels-387168c4.js

1import{j as e}from"./index-ab38db77.js";import{B as p}from"./BlogPost-920de3a8.js";import{U as h}from"./URDFArmVisualizer-bc31b749.js";import{A as t}from"./AdPlaceholder-9ddf71ab.js";import{C as a}from"./CopyableCodeBlock-51c83e81.js";import{A as x}from"./ArchitectureDiagramPane-291382c6.js";import{b as g}from"./blogArch-877cb2bf.js";import{S as o}from"./search-b7443866.js";import{M as r}from"./users-e8160425.js";import{L as l}from"./layers-3ca4a474.js";import{E as u}from"./eye-2c99b8de.js";import{A as n}from"./streakUtils-4796b5f1.js";import{B as i,Z as b}from"./Layout-ae221617.js";import{b as d}from"./cpu-1ebd5e89.js";import{A as f}from"./alert-triangle-194802c1.js";import{C as s}from"./check-circle-ebb31e13.js";import"./bookmarks-5e6598df.js";import"./TopicFollowButton-3468db39.js";import"./authRedirect-9431f409.js";import"./loader-2-1b0f84cc.js";import"./createLucideIcon-350a23d4.js";import"./check-826c63d6.js";import"./plus-0d9cf947.js";import"./BlogNavbar-43011ed4.js";import"./sheet-bd5a9df9.js";import"./index-b1c1ae7a.js";import"./index-1622c2bb.js";import"./index-74502314.js";import"./index-5f1a4c86.js";import"./index-38ea8514.js";import"./blogCategories-557cbee4.js";import"./arrow-left-78cc89c4.js";import"./book-open-d6b4bdaf.js";import"./user-plus-1a615a3a.js";import"./x-dbcb06f2.js";import"./target-1c7ddc66.js";import"./terminal-d75759e3.js";import"./code-2-e970a78e.js";import"./shield-check-89bf9027.js";import"./BlogFooter-0dec2c82.js";import"./mail-d6f89251.js";import"./badge-a8d68d49.js";import"./AnnotationPanel-a8595e5f.js";import"./trash-2-455aecb7.js";import"./save-a3420a34.js";import"./chevron-down-7df805dd.js";import"./chevron-up-ae8aca77.js";import"./customFlashcards-a28c57f8.js";import"./blogImages-9118753f.js";import"./urdfCatalog-dc9e6e1e.js";import"./glossaryData-ad15d072.js";import"./glossaryLinking-0d3c2fc9.js";import"./glossaryRegex-d95d90d2.js";import"./GlossaryLinker-5310e348.js";import"./glossaryRender-03caa63e.js";import"./react-katex-c8b55abe.js";import"./index-08c7c3df.js";import"./katex-a5db0322.js";import"./continueLearning-b7dee3d3.js";import"./lastReadPosition-3d456d91.js";import"./achievementToast-3307f533.js";import"./proxy-4188af44.js";import"./trophy-93bb35a2.js";import"./star-3fdb0fdf.js";import"./robobotClient-a6b235fb.js";import"./quizOptionBalancing-b8b82275.js";import"./EmbeddedQuizPractice-e895c045.js";import"./ExplanationRenderer-fd148189.js";/* empty css                  */import"./copy-61a61755.js";import"./prism-51c3acd6.js";import"./extends-4c19d496.js";import"./prism-9b2d42de.js";import"./index-3835e30c.js";import"./index.dom-42aa2b8a.js";import"./one-dark-2d2e2c01.js";import"./mistakeJournal-f029efa8.js";import"./message-circle-da57a6de.js";import"./x-circle-fdf008a7.js";import"./chevron-left-3f1f1bc1.js";import"./rotate-ccw-c11f09b7.js";import"./arrow-up-7e55e13f.js";import"./calendar-65001705.js";import"./maximize-2-92b2233e.js";import"./share-2-7aa2c950.js";import"./link-2-54f53ab8.js";import"./index-d6888bdf.js";import"./index-128356f7.js";import"./katex-991ac7df.js";import"./heart-16f11fbb.js";import"./minus-b0baa4e9.js";import"./button-aceb6222.js";import"./input-88a8d561.js";import"./bot-f88ad613.js";import"./studyVault-4c6970a9.js";const it=()=>{const c=e.jsxs("div",{className:"prose prose-lg max-w-none text-foreground",children:[e.jsxs("p",{className:"lead text-xl text-muted-foreground mb-8",children:["Vision-Language Models (VLMs) combine visual understanding with natural language processing, enabling robots to identify objects through descriptions, answer questions about scenes, and ground language to physical locations, the same grounding problem ",e.jsx("a",{href:"/learn/blog/saycan-benchmark-2026",className:"text-primary underline",children:"DeepMind's SayCan"})," formalized for action selection. In 2026 they have moved from research curiosities to production robot stacks, but choosing the right model for the right layer of your pipeline matters enormously."]}),e.jsx(t,{variant:"inline",className:"my-8"}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(o,{className:"w-6 h-6 text-blue-600"}),"CLIP: Contrastive Language-Image Pre-training"]}),e.jsxs("p",{children:[e.jsx("strong",{children:"CLIP"})," by OpenAI learns to match images with text descriptions using contrastive learning on internet-scale associations. For robots, this means ",e.jsx("strong",{children:"Open-Vocabulary Detection"}),": the robot does not need a fixed category list; it queries with natural language at inference time."]}),e.jsx(a,{code:`import torch
2import clip
3from PIL import Image
4
5# Load model
6device = "cuda" if torch.cuda.is_available() else "cpu"
7model, preprocess = clip.load("ViT-B/32", device=device)
8
9# Function to detect object from list
10def detect_object(image_path, queries):
11 image = preprocess(Image.open(image_path)).unsqueeze(0).to(device)
12 text = clip.tokenize(queries).to(device)
13
14 with torch.no_grad():
15 image_features = model.encode_image(image)
16 text_features = model.encode_text(text)
17
18 # Calculate similarity
19 logits_per_image, _ = model(image, text)
20 probs = logits_per_image.softmax(dim=-1).cpu().numpy()
21
22 # Get top prediction
23 best_idx = probs[0].argmax()
24 return queries[best_idx], probs[0][best_idx]
25
26# Usage
27# Robots don't need a "trained classifier" for "red mug".
28# They just ask: "Which label matches best?"
29labels = ["a red mug", "a blue pen", "an empty table", "a person"]
30label, conf = detect_object("camera_frame.jpg", labels)`,language:"python",filename:"clip_object_detection.py",className:"my-6"}),e.jsx("div",{className:"my-8 not-prose",children:e.jsx(x,{presetId:"cross-attn",customize:g.crossAttnVlm})}),e.jsx(t,{variant:"inline",className:"my-8"}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(r,{className:"w-6 h-6 text-purple-600"}),"LLaVA & GPT-4V"]}),e.jsxs("p",{children:[e.jsx("strong",{children:"LLaVA"}),' (Large Language and Vision Assistant) connects a vision encoder (like CLIP) to a 

30Large Language Model (like LLaMA). This allows the robot to "chat" about what it sees: asking it to explain a scene, diagnose a problem, or plan a sequence of steps.']}),e.jsxs("div",{className:"bg-slate-50 border border-slate-200 rounded-xl p-6 my-6 not-prose",children:[e.jsx("h4",{className:"font-bold mb-4",children:"Example Robot Query"}),e.jsxs("div",{className:"space-y-4",children:[e.jsxs("div",{className:"flex gap-4",children:[e.jsx("div",{className:"w-8 h-8 rounded-full bg-slate-500 flex items-center justify-center text-white text-xs font-bold",children:"USR"}),e.jsx("div",{className:"bg-white p-3 rounded-lg text-sm shadow-sm",children:'"I spilled my coffee. What should I do?"'})]}),e.jsxs("div",{className:"flex gap-4 justify-end",children:[e.jsx("div",{className:"bg-green-100 p-3 rounded-lg text-sm shadow-sm text-right",children:'"I see a brown liquid stain on the floor. I also see a paper towel roll on the counter. I should navigate to the counter, pick up the paper towels, and wipe the spill."'}),e.jsx("div",{className:"w-8 h-8 rounded-full bg-green-600 flex items-center justify-center text-white text-xs font-bold",children:"BOT"})]})]})]}),e.jsx(t,{variant:"inline",className:"my-8"}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(l,{className:"w-6 h-6 text-violet-600"}),"How VLMs Work Internally: The Encoder Projection Problem"]}),e.jsxs("p",{children:["The natural question when you first encounter a VLM is: how can a language model (trained entirely on text tokens) process an image at all? The answer is a deceptively simple idea called a ",e.jsx("strong",{children:"projection layer"}),", and understanding it explains both the power and the limits of every VLM you will deploy on a robot."]}),e.jsx("p",{className:"mt-4",children:"The core problem is that a vision encoder and a language model live in completely different vector spaces. A CLIP ViT-B/32 vision encoder produces a 768-dimensional embedding for every image patch. A 7B LLaMA-based language model expects 4096-dimensional token embeddings. These two spaces are not just different in size. They have entirely different geometry and semantics. You cannot concatenate them or average them. You need to teach one to speak the other's language."}),e.jsxs("div",{className:"bg-gradient-to-r from-slate-50 to-violet-50 border border-violet-100 rounded-2xl p-6 my-8 not-prose",children:[e.jsx("p",{className:"text-xs font-mono text-slate-400 mb-4 uppercase tracking-widest",children:"Inside a VLM: two worlds, one projection"}),e.jsxs("div",{className:"flex flex-col md:flex-row items-center gap-3 justify-center",children:[e.jsxs("div",{className:"bg-blue-600 text-white rounded-xl px-5 py-4 text-center shadow-md min-w-[140px]",children:[e.jsx(u,{className:"w-5 h-5 mx-auto mb-1"}),e.jsx("p",{className:"font-bold text-sm",children:"Vision Encoder"}),e.jsx("p",{className:"text-xs text-blue-200 mt-1",children:"ViT / CLIP"}),e.jsx("p",{className:"font-mono text-xs bg-blue-700 rounded mt-2 px-2 py-1",children:"768-dim"})]}),e.jsx("div",{className:"flex flex-col items-center gap-1",children:e.jsx(n,{className:"w-5 h-5 text-slate-400 rotate-90 md:rotate-0"})}),e.jsxs("div",{className:"bg-violet-600 text-white rounded-xl px-5 py-4 text-center shadow-md min-w-[140px] border-2 border-violet-400",children:[e.jsx(l,{className:"w-5 h-5 mx-auto mb-1"}),e.jsx("p",{className:"font-bold text-sm",children:"Projection MLP"}),e.jsx("p",{className:"text-xs text-violet-200 mt-1",children:"learned bridge"}),e.jsx("p",{className:"font-mono text-xs bg-violet-700 rounded mt-2 px-2 py-1",children:"768 → 4096"})]}),e.jsx("div",{className:"flex flex-col items-center gap-1",children:e.jsx(n,{className:"w-5 h-5 text-slate-400 rotate-90 md:rotate-0"})}),e.jsxs("div",{className:"bg-emerald-600 text-white rounded-xl px-5 py-4 text-center shadow-md min-w-[140px]",children:[e.jsx(i,{className:"w-5 h-5 mx-auto mb-1"}),e.jsx("p",{className:"font-bold text-sm",children:"Language Model"}),e.jsx("p",{className:"text-xs text-emerald-200 mt-1",children:"LLaMA / Mistral"}),e.jsx("p",{className:"font-mono text-xs bg-emerald-700 rounded mt-2 px-2 py-1",children:"4096-dim"})]}),e.jsx("div",{className:"flex flex-col items-center gap-1",children:e.jsx(n,{className:"w-5 h-5 text-slate-400 rotate-90 md:rotate-0"})}),e.jsxs("div",{className:"bg-orange-500 text-white rounded-xl px-5 py-4 text-center shadow-md min-w-[140px]",children:[e.jsx(r,{className:"w-5 h-5 mx-auto mb-1"}),e.jsx("p",{className:"font-bold text-sm",children:"Text Output"}),e.jsx("p",{className:"text-xs text-orange-100 mt-1",children:"description / plan"})]})]}),e.jsx("p",{className:"text-xs text-slate-500 text-center mt-4",children:"The LLM never sees raw pixels. It sees projected visual tokens that sit in its own embedding space alongside normal text tokens."})]}),e.jsx("h3",{className:"text-xl font-bold mt-8 mb-3",children:"LLaVA: The Clearest Example"}),e.jsx("p",{children:"LLaVA (Large Language and Vision Assistant) makes this concrete. Its architecture is three components bolted together: a frozen CLIP vision encoder, a two-layer MLP projection, and a frozen LLaMA language model. Only the projection MLP is trained end-to-e

30nd. The CLIP encoder stays frozen because it already knows what things look like. The LLM stays frozen because it already knows language. Training teaches the MLP to translate between them."}),e.jsx("p",{className:"mt-3",children:`The result: after training, a 256-token sequence of 4096-dim visual embeddings enters the LLM's attention layers alongside text tokens encoding "Describe what you see." The LLM has no special vision mode. It literally processes image patches as if they were words.`}),e.jsx(t,{variant:"sidebar",className:"my-8"}),e.jsx(a,{code:`import torch
31import torch.nn as nn
32from transformers import CLIPVisionModel, LlamaForCausalLM
33
34# The three components of a minimal VLM (LLaVA-style)
35class MinimalVLM(nn.Module):
36    def __init__(self):
37        # 1. Vision encoder, produces (B, num_patches, 768) embeddings
38        self.vision_encoder = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32")
39
40        # 2. Projection MLP, the only part that gets trained
41        #    Maps CLIP's 768-dim space → LLaMA's 4096-dim space
42        self.projection = nn.Sequential(
43            nn.Linear(768, 4096),
44            nn.GELU(),
45            nn.Linear(4096, 4096),
46        )
47
48        # 3. Language model, processes projected visual tokens like text
49        self.llm = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
50
51    def forward(self, images, input_ids):
52        # Step 1: Extract visual features, still in CLIP's 768-dim space
53        vision_out = self.vision_encoder(images)
54        patch_embeddings = vision_out.last_hidden_state  # (B, 256, 768)
55
56        # Step 2: Project into LLM's embedding space
57        visual_tokens = self.projection(patch_embeddings)  # (B, 256, 4096)
58
59        # Step 3: Get text token embeddings from the LLM
60        text_embeddings = self.llm.model.embed_tokens(input_ids)  # (B, T, 4096)
61
62        # Step 4: Concatenate, visual tokens come first, then text
63        # Now both live in the same 4096-dim space
64        combined = torch.cat([visual_tokens, text_embeddings], dim=1)  # (B, 256+T, 4096)
65
66        # Step 5: LLM processes the combined sequence normally
67        return self.llm(inputs_embeds=combined)`,language:"python",filename:"minimal_vlm.py",className:"my-6"}),e.jsx("h3",{className:"text-xl font-bold mt-8 mb-3",children:"Why This Matters for Robotics"}),e.jsxs("p",{children:["For robots, this architecture creates a direct consequence: the quality of your VLM's visual understanding is bounded by the ",e.jsx("strong",{children:"vision encoder's pretraining distribution"}),". CLIP was trained on internet images. If your robot's camera sees industrial fasteners, circuit boards, or surgical instruments under harsh lighting, the CLIP encoder may produce poor patch embeddings regardless of how good your projection MLP is. This is why robotics-specific VLMs like PaliGemma and Qwen2.5-VL fine-tune the vision encoder on domain data rather than freezing it."]}),e.jsxs("div",{className:"grid grid-cols-1 md:grid-cols-2 gap-4 my-6 not-prose",children:[e.jsxs("div",{className:"p-4 border border-blue-200 rounded-xl bg-blue-50",children:[e.jsx("h4",{className:"font-bold text-blue-900 mb-2 text-sm",children:"Frozen vision encoder (LLaVA-1.0)"}),e.jsx("p",{className:"text-xs text-blue-800",children:"Fast to train. Works well on everyday objects (cups, bottles, humans). Degrades on out-of-distribution robot scenes. Use for quick prototypes."})]}),e.jsxs("div",{className:"p-4 border border-violet-200 rounded-xl bg-violet-50",children:[e.jsx("h4",{className:"font-bold text-violet-900 mb-2 text-sm",children:"Fine-tuned vision encoder (LLaVA-1.5, Qwen2.5-VL)"}),e.jsx("p",{className:"text-xs text-violet-800",children:"More expensive to train. Handles domain-specific scenes. Necessary for industrial inspection, surgical robotics, or unusual lighting conditions."})]})]}),e.jsx(t,{variant:"inline",className:"my-8"}),e.jsx("div",{className:"my-10 not-prose",children:e.jsx(h,{initialPresetId:"turtlebot4",initialFamily:"mobile",title:"TurtleBot4 URDF, differential-drive wheeled mobile robot",locked:!0})}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(i,{className:"w-6 h-6 text-indigo-600"}),"Vision-Language-Action (VLA) Models"]}),e.jsxs("p",{children:["The next frontier is ",e.jsx("strong",{children:"VLA"}),". Instead of just describing a scene, these models output ",e.jsx("strong",{children:"actions"})," (motor commands) directly. They collapse perception, reasoning, and control into a single forward pass: with significant implications for latency and safety."]}),e.jsxs("div",{className:"grid grid-cols-1 md:grid-cols-2 gap-6 my-8 not-prose",children:[e.jsxs("div",{className:"bg-slate-50 p-6 rounded-xl border border-slate-200 ",children:[e.jsx("h4",{className:"font-bold text-indigo-900 mb-2",children:"RT-2 (Robotics Transformer 2)"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:`Google's vision-language-action model that co-trains on web data and robot trajectories. It can perform "emergent reasoning": like picking up a "dinosaur toy" even if it was never trained on that specific object.`})]}),e.jsxs("div",{className:"bg-slate-50 p-6 rounded-xl border border-slate-200 ",children:[e.jsx("h4",{className:"font-bold text-indigo-900 mb-2",children:"OpenVLA-7B"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:"A fully open 7-billion parameter VLA trained on 970K robot demonstrations from the Open X-Embodiment dataset. It achieves 85%+ success on manipulation tasks and fine-tunes on new robots with as few as 100 demonstrations."})]}),e.jsxs("div",{className:"bg-slate-50 p-6 rounded-xl border border-slate-200 ",children:[e.jsx("h4",{className:"font-bold text-indigo-900 mb-2",children:"PaLM-E"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:"An embodied multimodal language model that incorporates continuous sensor data (images, state estimates) into the LLM's token stream, allowing for high-level planning and low-level control in one model."})]}),e.jsxs("div",{className:"bg-slate-50 p-6 rounded-xl border border-slate-200 ",children:[e.jsx("h4",{className:"font-bold text-indigo-900 mb-2",children:"MolmoAct 2 (Ai2, 2026)"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:"Open VLA with flow-matching action head, adaptive depth reasoning, and 720+ hours of bimanual demonstrations. Reports 87.1% zero-shot success on Franka tasks at ~180 ms inference latency."})]})]}),e.jsx(t,{variant:"banner",className:"my-10"}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(o,{className:"w-6 h-6 text-blue-600"}),"Zero-Shot Detection & Grounding"]}),e.jsxs("p",{children:['How does a robot find "the half-eaten apple"? Traditional detectors fail here. ',e.jsx("strong",{children:"Grounding DINO"})," and ",e.jsx("strong",{children:"OWL-ViT"})," use VLMs to perform open-vocabulary object detection, returning bounding boxes for arbitrary text queries without any retraining."]}),e.jsxs("div",{className:"bg-slate-900 p-6 rounded-xl border border-slate-800 my-6 not-prose",children:[e.jsx("h4",{className:"text-blue-400 font-mono text-sm mb-4",children:"# Zero-Shot Grounding Logic"}),e.jsxs("ol",{className:"space-y-3 m-0 p-0 list-none",children:[e.jsxs("li",{className:"flex gap-3 items-start",children:[e.jsx("span",{className:"bg-slate-800 text-blue-400 text-xs w-6 h-6 rounded-full flex items-center justify-center shrink-0 mt-0.5 font-mono font-bold",children:"1"}),e.jsxs("p",{className:"text-slate-300 text-sm m-0",children:[e.jsx("strong",{className:"text-white",children:"Prompt:"}),' "Find the rusted bolt on the left engine block."']})]}),e.jsxs("li",{className:"flex gap-3 items-start",children:[e.jsx("span",{className:"bg-slate-800 text-blue-400 text-xs w-6 h-6 rounded-full flex items-center justify-center shrink-0 mt-0.5 font-mono font-bold",children:"2"}),e.jsxs("p",{className:"text-slate-300 text-sm m-0",children:[e.jsx("strong",{className:"text-white",children:"VLM:"})," Extracts visual features and text features."]})]}),e.jsxs("li",{className:"flex gap-3 items-start",children:[e.jsx("span",{className:"bg-slate-800 text-blue-400 text-xs w-6 h-6 rounded-full flex items-center justify-center shrink-0 mt-0.5 font-mono font-bold",children:"3"}),e.jsxs("p",{className:"text-slate-300 text-sm m-0",children:[e.jsx("strong",{className:"text-white",children:"Cross-Attention:"}),' Matches "rusted bolt" to specific pixels.']})]}),e.jsxs("li",{className:"flex gap-3 items-start",children:[e.jsx("span",{className:"bg-slate-800 text-blue-400 text-xs w-6 h-6 rounded-full flex items-center justify-center shrink-0 mt-0.5 font-mono font-bold",children:"4"}),e.jsxs("p",{className:"text-slate-300 text-sm m-0",children:[e.jsx("strong",{className:"text-white",children:"Output:"})," Bounding box + Confidence score."]})]})]})]}),e.jsxs("div",{className:"bg-amber-50 border border-amber-200 rounded-xl p-6 my-8 not-prose",children:[e.jsx("h4",{className:"font-bold text-amber-900 mb

67-2",children:"Where Zero-Shot Grounding Actually Breaks: Truly Novel Objects"}),e.jsxs("p",{className:"text-sm text-amber-900 mb-3",children:["The prompt-based pipeline above assumes the object has a describable name and that the VLM's training data covered something like it. Neither holds for a genuinely novel object, an odd-shaped fixture, a one-off tool, a part with no common name, and this isn't a hypothetical gap: the RAI Institute's ",e.jsx("a",{href:"https://rai-inst.com/resources/blog/show-dont-tell-detecting-novel-objects-by-watching-human-demonstrations/",target:"_blank",rel:"noopener noreferrer",className:"underline",children:`"Show, Don't Tell"`})," project reports VLM-based open-vocabulary detectors failing to find novel objects 43% of the time, even after five separate prompt attempts."]}),e.jsxs("p",{className:"text-sm text-amber-900 mb-3",children:["Their fix skips language entirely. A human picks up and manipulates the target object once, on camera; ",e.jsx("strong",{children:"HOIST-Former"})," detects the hand-object interaction, ",e.jsx("strong",{children:"SAMURAI"}),` (a SAM-based tracker) follows the object through the video before and after it's grasped, and that tracked footage is automatically turned into a labeled bounding-box dataset, no manual annotation. A lightweight Faster R-CNN detector fine-tuned on that single-demonstration dataset then becomes a fast, task-specific "manipulated objects detector." The reported numbers: the whole pipeline runs in under 10 minutes on a 15-second demonstration video, and the resulting detector beats GroundingDINO by 150% and RexOmni by 66% on the objects it was shown.`]}),e.jsx("p",{className:"text-sm text-amber-900",children:"The trade-off is the mirror image of zero-shot grounding's: no retraining needed vs. a few minutes of retraining required, arbitrary text queries vs. only objects actually demonstrated, general-purpose weights vs. a narrow, fast, task-specific detector. Zero-shot VLM grounding is still the right default when the object is nameable and roughly in-distribution; a one-shot fine-tuned detector is the better bet when it isn't."})]}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(d,{className:"w-6 h-6 text-cyan-600"}),"Running OpenVLA on a Jetson Robot"]}),e.jsx("p",{children:"OpenVLA-7B is the most practical open VLA for edge deployment in 2026. The snippet below shows how to load the model and run a single inference step with a camera image and a language instruction."}),e.jsx(a,{code:`from transformers import AutoModelForVision2Seq, AutoProcessor
68import torch
69from PIL import Image
70
71# Load model and processor (quantize for Jetson AGX Orin)
72model_id = "openvla/openvla-7b"
73processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
74model = AutoModelForVision2Seq.from_pretrained(
75 model_id,
76 attn_implementation="flash_attention_2",
77 torch_dtype=torch.bfloat16,
78 low_cpu_mem_usage=True,
79 trust_remote_code=True,
80).to("cuda:0")
81
82# Single inference step
83def get_robot_action(image: Image.Image, instruction: str) -> list[float]:
84 """
85 Returns 7-DoF action: [dx, dy, dz, droll, dpitch, dyaw, gripper]
86 """
87 inputs = processor(instruction, image).to("cuda:0", dtype=torch.bfloat16)
88 action = model.predict_action(**inputs, unnorm_key="bridge_orig", do_sample=False)
89 return action.tolist()
90
91# Usage
92img = Image.open("/dev/camera0.jpg")
93action = get_robot_action(img, "Pick up the red block and place it in the bin.")
94print(f"Action: {action}") # [dx, dy, dz, droll, dpitch, dyaw, gripper]`,language:"python",filename:"openvla_jetson.py",className:"my-6"}),e.jsx(t,{variant:"inline",className:"my-8"}),e.jsx("h2",{className:"text-2xl font-bold mt-12 mb-6 text-gray-900 ",children:"VLM Architectures in Robotics (2026)"}),e.jsx("div",{className:"overflow-x-auto my-8 not-prose",children:e.jsxs("table",{className:"w-full border-collapse text-sm text-left",children:[e.jsx("thead",{children:e.jsxs("tr",{className:"bg-slate-900 text-white",children:[e.jsx("th",{className:"p-4 font-bold rounded-tl-lg",children:"Model"}),e.jsx("th",{className:"p-4 font-bold",children:"Input"}),e.jsx("th",{className:"p-4 font-bold",children:"Output"}),e.jsx("th",{className:"p-4 font-bold",children:"Latency"}),e.jsx("th",{className:"p-4 font-bold rounded-tr-lg",children:"Primary Use Case"})]})}),e.jsxs("tbody",{className:"divide-y divide-slate-200",children:[e.jsxs("tr",{className:"bg-white ",children:[e.jsx("td",{className:"p-4 font-semibold text-slate-900 ",children:"CLIP (OpenAI)"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Image + Text"}),e.jsx("td",{className:"p-4 text-emerald-600 font-medium",children:"Similarity Score"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"<10 ms"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Open-vocab detection, semantic search"})]}),e.jsxs("tr",{className:"bg-slate-50 ",children:[e.jsx("td",{className:"p-4 font-semibold text-slate-900 ",children:"Grounding DINO"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Image + Query"}),e.jsx("td",{className:"p-4 text-emerald-600 font-medium",children:"Bounding Boxes"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"~80 ms"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Zero-shot object detection"})]}),e.jsxs("tr",{className:"bg-white ",children:[e.jsx("td",{className:"p-4 font-semibold text-slate-900 ",children:"LLaVA / GPT-4V"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Image + Prompt"}),e.jsx("td",{className:"p-4 text-emerald-600 font-medium",children:"Text (Reasoning)"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"500–2000 ms"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"High-level task planning, anomaly diagnosis"})]}),e.jsxs("tr",{className:"bg-slate-50 ",children:[e.jsx("td",{className:"p-4 font-semibold text-slate-900 ",children:"Qwen2.5-VL"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Image + Text"}),e.jsx("td",{className:"p-4 text-emerald-600 font-medium",children:"Text + Boxes"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"~300 ms"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Spatial grounding, reading dials and text"})]}),e.jsxs("tr",{className:"bg-white ",children:[e.jsx("td",{className:"p-4 font-semibold text-slate-900 ",children:"OpenVLA-7B"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Image + Instruction"}),e.jsx("td",{className:"p-4 text-emerald-600 font-medium",children:"Action (Motor 

94Commands)"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"~250–400 ms"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Direct low-level robot manipulation"})]}),e.jsxs("tr",{className:"bg-slate-50 ",children:[e.jsx("td",{className:"p-4 font-semibold text-slate-900 ",children:"MolmoAct 2"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Image + Instruction"}),e.jsx("td",{className:"p-4 text-emerald-600 font-medium",children:"Action Chunk (Flow)"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"~180 ms"}),e.jsx("td",{className:"p-4 text-slate-700 ",children:"Bimanual manipulation, open-source eval"})]})]})]})}),e.jsx(t,{variant:"banner",className:"my-10"}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(o,{className:"w-6 h-6 text-green-600"}),"VLM-based Semantic Navigation"]}),e.jsxs("p",{children:["Instead of navigating to (x, y) coordinates, robots can now navigate to ",e.jsx("strong",{children:"semantic descriptions"}),".",e.jsx("em",{children:'"Go to the room with the red sofa and wait by the coffee table."'}),"VLMs allow the robot to build a ",e.jsx("strong",{children:"Semantic Map"})," where every location is associated with a language description, enabling intuitive human-robot interaction. Models like ",e.jsx("strong",{children:"SayNav"})," and ",e.jsx("strong",{children:"LM-Nav"})," demonstrate robots navigating km-scale outdoor paths using only a language description and satellite imagery."]}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(b,{className:"w-6 h-6 text-amber-500"}),"Depth and 3D Spatial Reasoning"]}),e.jsxs("p",{children:["A key limitation of early VLMs was flat, 2D understanding. 2026 models increasingly incorporate depth: either from RGB-D cameras or monocular depth estimation: to reason about 3D structure. ",e.jsx("strong",{children:"MolmoAct 2"})," uses adaptive depth routing: depth tokens are only activated when the task requires precise 3D structure (e.g., stacking, pouring), keeping latency low for simpler tasks."]}),e.jsxs("div",{className:"grid grid-cols-1 md:grid-cols-2 gap-4 my-6 not-prose",children:[e.jsxs("div",{className:"p-4 border border-slate-200 rounded-lg bg-slate-50 ",children:[e.jsx("h4",{className:"font-bold mb-2",children:"Monocular Depth"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:"Models like Depth Anything V2 estimate metric depth from a single RGB frame. Adds 3D context without a dedicated depth sensor: useful for low-cost manipulation setups."})]}),e.jsxs("div",{className:"p-4 border border-slate-200 rounded-lg bg-slate-50 ",children:[e.jsx("h4",{className:"font-bold mb-2",children:"RGB-D Fusion"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:'Structured-light or ToF cameras provide metric depth. Fused with VLM embeddings, the robot can reason about exact distances: "the cup is 18 cm in front of the hand."'})]}),e.jsxs("div",{className:"p-4 border border-slate-200 rounded-lg bg-slate-50 ",children:[e.jsx("h4",{className:"font-bold mb-2",children:"3D Scene Graphs"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:"Combining VLM grounding with 3D point clouds allows robots to build spatial relationship graphs: object X is left of Y, on top of Z: enabling compositional manipulation reasoning."})]}),e.jsxs("div",{className:"p-4 border border-slate-200 rounded-lg bg-slate-50 ",children:[e.jsx("h4",{className:"font-bold mb-2",children:"Affordance + Depth"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:'Combining VLM affordance prediction ("grasp the handle") with depth maps gives the robot a 3D grasp pose hypothesis: bridging high-level VLM output to low-level motor commands.'})]})]}),e.jsx(t,{variant:"inline",className:"my-8"}),e.jsx("h2",{className:"text-2xl font-bold mt-12 mb-6",children:"Robotic Capabilities Enabled by VLMs"}),e.jsxs("div",{className:"grid grid-cols-1 md:grid-cols-2 gap-4 my-6 not-prose",children:[e.jsxs("div",{className:"p-4 border border-slate-200 rounded-lg",children:[e.jsx("h4",{className:"font-bold mb-2",children:"Visual Grounding"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:'Mapping "the blue cup to the left of the laptop" to specific pixel coordinates.'})]}),e.jsxs("div",{className:"p-4 border border-slate-200 rounded-lg",children:[e.jsx("h4",{className:"font-bold mb-2",children:"Affordance Detection"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:'Understanding that a "cup" can be "grasped" by the handle, without explicit training.'})]}),e.jsxs("div",{className:"p-4 border border-slate-200 rounded-lg",children:[e.jsx("h4",{className:"font-bold mb-2",children:"Anomaly Detection"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:'Identifying "a loose wire" or "smoke" in an industrial inspection task using natural language prompts.'})]}),e.jsxs("div",{className:"p-4 border border-slate-200 rounded-lg",children:[e.jsx("h4",{className:"font-bold mb-2",children:"Zero-Shot Learning"}),e.jsx("p",{className:"text-sm text-muted-foreground",children:"Recognizing thousands of object categories not seen during robot training."})]})]}),e.jsx(t,{variant:"inline",className:"my-8"}),e.jsx("h2",{className:"text-2xl font-bold mt-12 mb-6",children:"VLM Model Selection by Task Type"}),e.jsx("p",{className:"mb-4",children:"Choosing the wrong VLM layer is a common cause of slow or unstable robot pipelines. Use this table to match model type to your specific robot task."}),e.jsx("div",{className:"overflow-x-auto my-8 not-prose",children:e.jsxs("table",{className:"w-full border-collapse text-sm",children:[e.jsx("thead",{children:e.jsxs("tr",{className:"bg-slate-900 text-white",children:[e.jsx("th",{className:"p-3 text-left",children:"Task"}),e.jsx("th",{className:"p-3 text-left",children:"Recommended Model"}),e.jsx("th",{className:"p-3 text-left",children:"Why"}),e.jsx("th",{className:"p-3 text-left",children:"Latency"})]})}),e.jsxs("tbody",{children:[e.jsxs("tr",{className:"border-b bg-white",children:[e.jsx("td",{className:"p-3 font-semibold",children:"Open-vocab object detection"}),e.jsx("td",{className:"p-3",children:"Grounding DINO + SAM"}),e.jsx("td",{className:"p-3",children:"Returns bounding boxes for any text query, no retraining"}),e.jsx("td",{className:"p-3",children:"80–120 ms"})]}),e.jsxs("tr",{className:"border-b bg-slate-50",children:[e.jsx("td",{className:"p-3 font-semibold",children:"Semantic classification"}),e.jsx("td",{className:"p-3",children:"CLIP ViT-B/32"}),e.jsx("td",{className:"p-3",children:"Fastest option; ranks labels by cosine similarity"}),e.jsx("td",{className:"p-3",children:"<10 ms"})]}),e.jsxs("tr",{className:"border-b bg-white",children:[e.jsx("td",{className:"p-3 font-semibold",children:"Scene description / planning"}),e.jsx("td",{className:"p-3",children:"LLaVA-Next or Qwen2.5-VL"}),e.jsx("td",{className:"p-3",children:"Rich language output for task reasoning"}),e.jsx("td",{className:"p-3",children:"300–800 ms"})]}),e.jsxs("tr",{className:"border-b bg-slate-50",children:[e.jsx("td",{className:"p-3 font-semibold",children:"Direct manipulation action"}),e.jsx("td",{className:"p-3",children:"OpenVLA-7B or MolmoAct 2"}),e.jsx("td",{className:"p-3",children:"Outputs motor commands directly from image + instruction"}),e.jsx("td",{className:"p-3",children:"180–400 ms"})]}),e.jsxs("tr",{className:"border-b bg-white",children:[e.jsx("td",{className:"p-3 font-semibold",children:"Reading gauges / dials / text"}),e.jsx("td",{className:"p-3",children:"Qwen2.5-VL or InternVL2"}),e.jsx("td",{className:"p-3",children:"Strong OCR and spatial grounding for industrial inspection"}),e.jsx("td",{className:"p-3",children:"200–500 ms"})]}),e.jsxs("tr",{className:"bg-slate-50",children:[e.jsx("td",{className:"p-3 font-semibold",children:"Success / reward checking"}),e.jsx("td",{className:"p-3",children:"GPT-4o or Gemini 1.5"}),e.jsx("td",{className:"p-3",children:"Reliable binary yes/no judgment on task completion from image"}),e.jsx("td",{className:"p-3",children:"500–2000 ms"})]})]})]})}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(d,{className:"w-6 h-6 text-teal-600"}),"Running CLIP in a ROS 2 Node"]}),e.jsx("p",{className:"mb-4",children:"Here is a minimal ROS 2 node that subscribes to a camera topic and publishes the best-matching label using CLIP. This is the fastest open-vocabulary detection pattern and runs well on a Jetson Orin."}),e.jsx(a,{code:`import rclpy
95from rclpy.node import Node
96from sensor_msgs.msg import Image
97from std_msgs.msg import String
98from cv_bridge import CvBridge
99import torch, clip
100from PIL import Image as PILImage
101import numpy as np
102
103class CLIPClassifierNode(Node):
104 def __init__(self):
105 super().__init__('clip_classifier')
106 self.bridge = CvBridge()
107 self.device = "cuda" if torch.cuda.is_available() else "cpu"
108 self.model, self.preprocess = clip.load("ViT-B/32", device=self.device)
109
110 self.labels = self.declare_parameter(
111 'labels', ['red block', 'blue cup', 'empty table', 'person']
112 ).value
113 self.text = clip.tokenize(self.labels).to(self.device)
114
115 self.sub = self.create_subscription(Image, '/camera/rgb', self.cb, 1)
116 self.pub = self.create_publisher(String, '/clip/best_label', 10)
117
118 def cb(self, msg):
119 cv_img = self.bridge.imgmsg_to_cv2(msg, desired_encoding='rgb8')
120 pil_img = PILImage.fromarray(cv_img)
121 image = self.preprocess(pil_img).unsqueeze(0).to(self.device)
122
123 with torch.no_grad():
124 logits, _ = self.model(image, self.text)
125 probs = logits.softmax(dim=-1)[0].cpu().numpy()
126
127 best = self.labels[probs.argmax()]
128 self.pub.publish(String(data=best))
129
130def main():
131 rclpy.init()
132 rclpy.spin(CLIPClassifierNode())
133 rclpy.shutdown()`,language:"python"}),e.jsx(t,{variant:"inline",className:"my-8"}),e.jsxs("h2",{className:"text-2xl font-bold mt-12 mb-6 flex items-center gap-2",children:[e.jsx(f,{className:"w-6 h-6 text-amber-500"}),"VLM Deployment Checklist for Robotics"]}),e.jsx("div",{className:"bg-amber-50 border border-amber-200 rounded-xl p-6 my-6 not-prose",children:e.jsxs("ul",{className:"space-y-3 text-sm text-amber-900",children:[e.jsxs("li",{className:"flex gap-2",children:[e.jsx(s,{className:"w-4 h-4 mt-0.5 shrink-0 text-amber-700"}),e.jsxs("span",{children:[e.jsx("strong",{children:"Latency budget:"})," Define the maximum acceptable inference time. Use CLIP/Grounding DINO for fast loops (<100 ms), LLaVA for slower reasoning steps (500+ ms)."]})]}),e.jsxs("li",{className:"flex gap-2",children:[e.jsx(s,{className:"w-4 h-4 mt-0.5 shrink-0 text-amber-700"}),e.jsxs("span",{children:[e.jsx("strong",{children:"Output format:"})," Decide if you need text, bounding boxes, or action tokens: this drives your model choice more than raw benchmark scores."]})]}),e.jsxs("li",{className:"flex gap-2",children:[e.jsx(s,{className:"w-4 h-4 mt-0.5 shrink-0 text-amber-700"}),e.jsxs("span",{children:[e.jsx("strong",{children:"Safety wrapper:"})," VLM outputs are not verified motor commands. Always add workspace limits, force thresholds, and a watchdog before sending actions to hardware."]})]}),e.jsxs("li",{className:"flex gap-2",children:[e.jsx(s,{className:"w-4 h-4 mt-0.5 shrink-0 text-amber-700"}),e.jsxs("span",{children:[e.jsx("strong",{children:"OOD robustness:"})," Test on objects, lighting, and prompts not in your demo set before deployment. VLMs can confidently hallucinate in novel scenes."]})]}),e.jsxs("li",{className:"flex gap-2",children:[e.jsx(s,{className:"w-4 h-4 mt-0.5 shrink-0 text-amber-700"}),e.jsxs("span",{children:[e.jsx("strong",{children:"Camera calibration:"})," Bounding boxes from VLMs are only useful if you know the camera-to-robot transform. A miscalibrated hand-eye relationship will cause consistent grasping failures regardless of model quality."]})]})]})}),e.jsx(t,{variant:"sidebar",className:"my-8"}),e.jsxs("div",{className:"bg-gradient-to-r from-indigo-500 to-purple-600 text-white p-8 rounded-2xl my-12 not-prose shadow-lg",children:[e.jsxs("div",{className:"flex items-center gap-3 mb-4",children:[e.jsx(i,{className:"w-6 h-6 text-white"}),e.jsx("h3",{className:"text-2xl font-bold mb-0",children:"The Future"})]}),e.jsx("p",{className:"text-indigo-100",children:'VLMs are the bridge between "perceiving pixels" and "understanding concepts." They are the key to building general-purpose robots that can operate in unstructured human environments, but they are tools, not silver bullets. The winning 2026 pattern is a layered stack: fast open-vocabulary detectors for real-time loops, reasoning VLMs for task planning, and VLA models for action generation, each validated separately and composed with explicit safety boundaries.'})]})]}),m=[{title:"OpenAI CLIP Paper",url:"https://openai.com/research/clip",type:"paper"},{title:"OpenVLA: Open-Source Vision-Language-Action Model",url:"https://arxiv.org/abs/2406.09246",type:"paper"},{title:"Grounding DINO: Open-Set Object Detection",url:"https://arxiv.org/abs/2303.05499",type:"paper"},{title:"LLaVA Project Page",url:"https://llava-vl.github.io/",type:"tech"},{title:"Segment Anything Model (SAM)",url:"https://segment-anything.com/",type:"tech"},{title:"Show, Don't Tell: Detecting Novel Objects by Watching Human Demonstrations (RAI Institute)",url:"https://rai-inst.com/resources/blog/show-dont-tell-detecting-novel-objects-by-watching-human-demonstrations/",type:"tech"}];return e.jsx(p,{title:"Vision-Language Models for Robotics: 2026 Complete Guide",seoTitle:"Vision-Language Models for Robotics: CLIP, LLaVA, OpenVLA",seoDescription:"Vision-language models for robotics explained: CLIP zero-shot detection, LLaVA projection layers, OpenVLA action generation, VLM vs VLA, and model selection.",description:"Learn: using CLIP, LLaVA, OpenVLA, and multimodal models for robot perception, natural language grounding, zero-shot detection, projection layer architecture, and VLA action generation.",date:"Dec 17, 2025",author:"Darsh Menon",readTime:"32 min",category:"AI & Perception",lastUpdated:"Jun 24, 2026",difficulty:"Advanced",wordCount:2700,faqs:[{question:"What is Vision-Language Models for Robotics?",answer:"VLMs combine image and text understanding to let robots identify objects through natural language, navigate to semantic descriptions, detect anomalies, and generate motor actions: all without per-object training."},{question:"How do Vision-Language Models work internally?",answer:"A VLM has three parts: a frozen vision encoder (e.g. CLIP ViT) that converts image patches into embeddings, a trained projection MLP that maps those embeddings into the LLM's token space, and a language model that processes visual tokens alongside text tokens. Only the projection MLP is trained, the encoder and LLM can stay frozen, which is why VLMs are fast to fine-tune."},{question:"How is OpenVLA used in robotics?",answer:"OpenVLA-7B is a 7B-parameter vision-language-action model trained on 970K robot demonstrations. Given a camera image and a language instruction, it outputs a 7-DoF action vector (delta x, y, z, roll, pitch, yaw, gripper) at ~250–400 ms latency, making it suitable for tabletop manipulation robots."},{question:"What is the difference between a VLM and a VLA?",answer:"A VLM maps images and language to semantic outputs: text descriptions, bounding boxes, or reasoning. A VLA (Vision-Language-Action) model additionally outputs robot actions. VLMs are used for perception and planning;

133 VLAs are used for end-to-end manipulation control."},{question:"Which VLM should I use for my robot in 2026?",answer:"Use CLIP or Grounding DINO for fast open-vocabulary detection (<100 ms). Use LLaVA or Qwen2.5-VL for slow scene reasoning and planning (300–2000 ms). Use OpenVLA-7B or MolmoAct 2 when you need the model to output motor commands directly. If your domain is industrial or surgical, use a model with a fine-tuned vision encoder (LLaVA-1.5 or Qwen2.5-VL) rather than a frozen CLIP backbone."},{question:"When should I fine-tune the vision encoder in a VLM?",answer:"Fine-tune the vision encoder when your robot operates in an out-of-distribution visual domain, industrial fasteners, circuit boards, surgical instruments, or unusual lighting. Frozen CLIP encoders (used in LLaVA-1.0) degrade on scenes underrepresented in internet-scale training data. Fine-tuned encoders (LLaVA-1.5, Qwen2.5-VL, PaliGemma) handle these cases significantly better."},{question:"What do I do when a robot needs to detect an object with no describable name?",answer:`Zero-shot VLM grounding (Grounding DINO, OWL-ViT) needs the object to be nameable and roughly in-distribution; the RAI Institute's "Show. Don't Tell" project reports these detectors failing on truly novel objects 43% of the time even after five prompt attempts. Their alternative: have a human manipulate the object once on camera, auto-label it via hand-object interaction tracking (HOIST-Former + SAMURAI), and fine-tune a lightweight detector (Faster R-CNN) on that single demonstration, under 10 minutes end to end, and reported to beat Grounding DINO by 150% and RexOmni by 66% on the demonstrated objects.`}],relatedLinks:[{title:"Tokenizing Robot Actions: Quantile Binning, Vocabulary Reuse, and Why 256 Bins",link:"/learn/blog/action-tokenization-robot-policies",category:"Foundation Models",description:"How the action side of a VLA gets tokenized into the same vocabulary space this post's vision-language tokens live in."},{title:"RoPE: Rotary Positional Embeddings, and Why Relative Distance Is What Actually Matters",link:"/learn/blog/rope-rotary-positional-embeddings",category:"Foundation Models",description:"The positional encoding scheme behind the transformer backbones VLMs use to attend over image and language tokens."},{title:"Vision-Language Models (Interactive Lessons)",link:"/robotics-concepts/vision-language-models",category:"Interactive",description:"CLIP encoders, open-vocabulary grounding, and VLM-to-VLA action prediction as a hands-on course."},{title:"Modern AI (Interactive Lessons)",link:"/robotics-concepts/modern-ai",category:"Interactive",description:"VLMs (CLIP, BLIP-2, LLaVA, PaLI) as the vision-language backbone of modern robot policies."},{title:"Computer Vision (Interactive Lessons)",link:"/robotics-concepts/computer-vision",category:"Interactive",description:"Visual encoders in VLMs: ViT, CLIP, and contrastive pretraining for robot scene understanding."},{title:"VLMs for Robot Control Survey",link:"/learn/blog/vision-language-models-robot-control-survey",category:"Blog",description:"Survey of how VLMs are integrated into robot control stacks for instruction-following."},{title:"Point Cloud Processing for Robotics",link:"/learn/blog/point-cloud-processing",category:"Blog",description:"The 3D point clouds that VLM grounding is combined with to build spatial scene graphs."},{title:"Foundation Models for Robotics",link:"/learn/blog/foundation-models-robotics",category:"Blog",description:"Robot foundation models using VLMs as the vision-language backbone for action generation."},{title:"DeepMind RT-2 Architecture",link:"/learn/blog/deepmind-rt2-architecture",category:"Blog",description:"RT-2 as a VLM fine-tuned for robot action prediction: the seminal VLA architecture."}],content:c,image:"/assets/images/generated/vision_language_models_robotics_hero.webp",canonicalUrl:"https://robocloud-dashboard.vercel.app/learn/blog/vision-language-models",keywords:["VLM","Vision-Language Models","CLIP","OpenVLA","LLaVA","Multimodal AI","Robotics","Grounded Language","VLA","SOTA AI 2026","LLaVA architecture","vision encoder projection layer","how VLMs work","CLIP ViT robotics","frozen vs fine-tuned vision encoder","Qwen2.5-VL robotics"],relatedResources:m})};export{it as default};