1"use strict";(self.webpackChunkplaywright_dev=self.webpackChunkplaywright_dev||[]).push([["3941"],{92271(e,s,i){i.r(s),i.d(s,{metadata:()=>n,default:()=>h,frontMatter:()=>r,contentTitle:()=>a,toc:()=>l,assets:()=>d});var n=JSON.parse('{"id":"vision-mode","title":"Vision Mode","description":"By default, Playwright MCP uses accessibility snapshots for all interactions. Vision mode adds coordinate-based tools that work with screenshots, enabling interaction with elements not exposed in the accessibility tree.","source":"@site/mcp/vision-mode.mdx","sourceDirName":".","slug":"/vision-mode","permalink":"/mcp/vision-mode","draft":false,"unlisted":false,"tags":[],"version":"current","frontMatter":{"title":"Vision Mode"},"sidebar":"mcp","previous":{"title":"Capabilities","permalink":"/mcp/capabilities"},"next":{"title":"Navigation","permalink":"/mcp/tools/navigation"}}'),t=i(74848),o=i(28453);let r={title:"Vision Mode"},a="Vision Mode",d={},l=[{value:"Enabling vision mode",id:"enabling-vision-mode",level:2},{value:"Additional tools",id:"additional-tools",level:2},{value:"Workflow: interacting with a canvas app",id:"workflow-interacting-with-a-canvas-app",level:2},{value:"Workflow: clicking an icon without accessible name",id:"workflow-clicking-an-icon-without-accessible-name",level:2},{value:"When to use vision mode",id:"when-to-use-vision-mode",level:2},{value:"Combining capabilities",id:"combining-capabilities",level:2}];function c(e){let s={a:"a",admonition:"admonition",code:"code",h1:"h1",h2:"h2",header:"header",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",...(0,o.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(s.header,{children:(0,t.jsx)(s.h1,{id:"vision-mode",children:"Vision Mode"})}),"\n",(0,t.jsx)(s.p,{children:"By default, Playwright MCP uses accessibility snapshots for all interactions. Vision mode adds coordinate-based tools that work with screenshots, enabling interaction with elements not exposed in the accessibility tree."}),"\n",(0,t.jsx)(s.h2,{id:"enabling-vision-mode",children:"Enabling vision mode"}),"\n",(0,t.jsxs)(s.p,{children:["Add the ",(0,t.jsx)(s.code,{children:"vision"})," capability:"]}),"\n",(0,t.jsx)(s.pre,{children:(0,t.jsx)(s.code,{className:"language-json",children:'{\n "mcpServers": {\n "playwright": {\n "command": "npx",\n "args": ["@playwright/mcp@latest", "--caps=vision"]\n }\n }\n}\n'})}),"\n",(0,t.jsx)(s.h2,{id:"additional-tools",children:"Additional tools"}),"\n",(0,t.jsx)(s.p,{children:"With vision mode, these coordinate-based tools become available:"}),"\n",(0,t.jsxs)(s.table,{children:[(0,t.jsx)(s.thead,{children:(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.th,{children:"Tool"}),(0,t.jsx)(s.th,{children:"Description"})]})}),(0,t.jsxs)(s.tbody,{children:[(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:(0,t.jsx)(s.code,{children:"browser_mouse_move_xy"})}),(0,t.jsx)(s.td,{children:"Move mouse to x,y coordinates"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:(0,t.jsx)(s.code,{children:"browser_mouse_click_xy"})}),(0,t.jsx)(s.td,{children:"Click at x,y (supports button, clickCount, delay)"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:(0,t.jsx)(s.code,{children:"browser_mouse_drag_xy"})}),(0,t.jsx)(s.td,{children:"Drag from start to end coordinates"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:(0,t.jsx)(s.code,{children:"browser_mouse_down"})}),(0,t.jsx)(s.td,{children:"Press mouse button"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:(0,t.jsx)(s.code,{children:"browser_mouse_up"})}),(0,t.jsx)(s.td,{children:"Release mouse button"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:(0,t.jsx)(s.code,{children:"browser_mouse_wheel"})}),(0,t.jsx)(s.td,{children:"Scroll with mouse wheel"})]})]})]}),"\n",(0,t.jsxs)(s.p,{children:["See ",(0,t.jsx)(s.a,{href:"/mcp/tools/keyboard-mouse",children:"Keyboard & Mouse"})," for the full parameter reference."]}),"\n",(0,t.jsx)(s.h2,{id:"workflow-interacting-with-a-canvas-app",children:"Workflow: interacting with a canvas app"}),"\n",(0,t.jsx)(s.pre,{children:(0,t.jsx)(s.code,{className:"language-txt",children:"You: Draw a rectangle on the canvas.\n\n\u2192 browser_take_screenshot\n // LLM sees the canvas and identifies coordinates\n\n\u2192 browser_mouse_click_xy { x: 100, y: 150 }\n\u2192 browser_mouse_drag_xy { startX: 100, startY: 150, endX: 300, endY: 250 }\n\u2192 browser_take_screenshot\n // LLM verifies the rectangle was drawn\n"})}),"\n",(0,t.jsx)(s.h2,{id:"workflow-clicking-an-icon-without-accessible-name",children:"Workflow: clicking an icon without accessible name"}),"\n",(0,t.jsx)(s.pre,{children:(0,t.jsx)(s.code,{className:"language-txt",children:'\u2192 browser_snapshot\n // The gear icon has no accessible name in the snapshot\n\n\u2192 browser_take_screenshot\n // LLM sees the gear icon at approximately (850, 45)\n\n\u2192 browser_mouse_click_xy { x: 850, y: 45 }\n\n // Settings panel is now open with proper accessibility\n - heading "Settings" [level=2] [ref=e10]\n - textbox "Display name" [ref=e12]\n'})}),"\n",(0,t.jsx)(s.admonition,{type:"tip",children:(0,t.jsxs)(s.p,{children:["Instead of reading coordinates off a screenshot, ask for them directly:\n",(0,t.jsx)(s.code,{children:"browser_snapshot { boxes: true }"})," adds ",(0,t.jsx)(s.code,{children:"[box=x,y,width,height]"})," to every node, viewport-relative in\nCSS pixels. ",(0,t.jsx)(s.code,{children:"--snapshot-boxes"})," turns this on for every snapshot."]})}),"\n",(0,t.jsx)(s.h2,{id:"when-to-use-vision-mode",children:"When to use vision mode"}),"\n",(0,t.jsxs)(s.table,{children:[(0,t.jsx)(s.thead,{children:(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.th,{children:"Scenario"}),(0,t.jsx)(s.th,{children:"Approach"})]})}),(0,t.jsxs)(s.tbody,{children:[(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"Standard web pages"}),(0,t.jsx)(s.td,{children:"Use refs from snapshots (default)"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"Canvas / WebGL apps"}),(0,t.jsx)(s.td,{children:"Vision mode with screenshots"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"Map interactions"}),(0,t.jsx)(s.td,{children:"Vision mode for pan/zoom"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"Image editors"}),(0,t.jsx)(s.td,{children:"Vision mode for drawing"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"Charts / graphs"}),(0,t.jsx)(s.td,{children:"Vision mode to click data points"})]}),(0,t.jsxs)(s.tr,{children:[(0,t.jsx)(s.td,{children:"Custom widgets without ARIA"}),(0,t.jsx)(s.td,{children:"Vision mode as fallback"})]})]})]}),"\n",(0,t.jsxs)(s.p,{children:["For most web applications, ",(0,t.jsx)(s.strong,{children:"the default snapshot-based approach is more reliable and token-efficient"}),". Use vision mode only when the accessibility tree doesn't cover your use case."]}),"\n",(0,t.jsx)(s.h2,{id:"combining-capabilities",children:"Combining capabilities"}),"\n",(0,t.jsx)(s.p,{children:"Enable multiple capabilities:"}),"\n",(0,t.jsx)(s.pre,{children:(0,t.jsx)(s.code,{className:"language-json",children:'{\n "mcpServers": {\n "playwright": {\n "command": "npx",\n "args": ["@playwright/mcp@latest", "--caps=vision,pdf,devtools"]\n }\n }\n}\n'})}),"\n",(0,t.jsx)(s.p,{children:"Or in the config file:"}),"\n",(0,t.jsx)(s.pre,{children:(0,t.jsx)(s.code,{className:"language-json",children:'{\n "capabilities": ["vision", "pdf", "devtools"]\n}\n'})})]})}function h(e={}){let{wrapper:s}={...(0,o.R)(),...e.components};return s?(0,t.jsx)(s,{...e,children:(0,t.jsx)(c,{...e})}):c(e)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.