1/** 2 * AI Search â CLIP-powered semantic search for events. 3 * 4 * Privacy: All CLIP inference runs on the user's device. No video or 5 * derived data ever leaves the browser. 6 * 7 * Public API (window.aiSearch): 8 * state â reactive state object 9 * enable(opts) â first-time enable + index all events 10 * indexAll(opts) â re-index everything 11 * search(query) â returns ranked results with tier metadata 12 * deepIndex(eventId) â re-index one event with motion strategy 13 * getTags() â returns category â event count map 14 * clearIndex() â wipe everything 15 * 16 * Events dispatched on `window` for UI integration: 17 * 'ai-search:status' â detail: { state, message } 18 * 'ai-search:progress' â detail: { done, total, status } 19 * 'ai-search:ready' â when CLIP is loaded and index is ready 20 * 'ai-search:results' â after a search runs 21 */ 22(function () { 23 'use strict'; 24 25 // ---- Configuration ---------------------------------------------------- 26 const CLIP_MODELS = { 27 'base': { id: 'Xenova/clip-vit-base-patch32', name: 'Fast (CLIP ViT-B/32, 175 MB)', dim: 512, thresholds: { confident: 0.08, possible: 0.04, tag: 0.03 } }, 28 'large': { id: 'Xenova/clip-vit-large-patch14', name: 'Quality (CLIP ViT-L/14, ~570 MB)', dim: 768, thresholds: { confident: 0.04, possible: 0.02, tag: 0.015 } } 29 }; 30 const SENTRY_OFFSETS = [-4, -2, 0, 2, 4]; 31 const CAMERA_ID_MAP = { '0': 'front', '1': 'back', '4': 'left_pillar', '5': 'left_repeater', '6': 'right_repeater', '7': 'right_pillar' }; 32 33 const QUERY_TEMPLATES = [ 34 q => `a photo of ${q}`, 35 q => `a dashcam photo of ${q}`, 36 q => `a picture of ${q} taken from a car`, 37 q => `a frame from a dashboard camera showing ${q}`, 38 q => `a Tesla dashcam image of ${q}` 39 ]; 40 const NEUTRAL_PROMPTS = ['a photo', 'an image', 'a picture', 'a dashcam photo', 'a frame from a dashboard camera']; 41 42 const SCENE_CATEGORIES = [ 43 { key: 'night-drive', label: 'Night drive', keywords: ['night drive', 'night driving', 'dark drive', 'driving at night'],
44 prompts: ['a dashcam photo of a night drive', 'driving at night', 'headlights on a dark road'] }, 45 { key: 'daytime-drive', label: 'Daytime drive', keywords: ['daytime drive', 'day drive', 'sunny drive'], 46 prompts: ['a dashcam photo of driving during the day', 'daytime driving scene', 'sunny daytime road'] }, 47 { key: 'highway', label: 'Highway', keywords: ['highway', 'freeway', 'interstate'], 48 prompts: ['a photo of a highway', 'driving on a freeway', 'multi-lane highway scene'] }, 49 { key: 'residential', label: 'Residential', keywords: ['residential', 'neighborhood', 'suburban'], 50 prompts: ['a residential neighborhood street', 'houses along a suburban street'] }, 51 { key: 'parking-garage', label: 'Parking garage', keywords: ['parking garage', 'garage', 'parking deck'], 52 prompts: ['a photo of a parking garage', 'indoor parking structure'] }, 53 { key: 'parking-lot', label: 'Parking lot', keywords: ['parking lot', 'lot'], 54 prompts: ['a photo of a parking lot', 'outdoor parking area'] }, 55 { key: 'driveway', label: 'Driveway / home', keywords: ['driveway', 'home'], 56 prompts: ['a driveway at a house', 'parked in a driveway'] }, 57 { key: 'pedestrian', label: 'Pedestrian', keywords: ['pedestrian', 'person', 'people', 'walker', 'walking'], 58 prompts: ['a pedestrian walking', 'a person walking down the street', 'person on sidewalk'] }, 59 { key: 'traffic-lights', label: 'Traffic lights', keywords: ['traffic light', 'traffic lights', 'stoplight'], 60 prompts: ['a photo of traffic lights', 'intersection with traffic lights'] }, 61 { key: 'multiple-vehicles',label: 'Many vehicles', keywords: ['many cars', 'many vehicles', 'traffic'], 62 prompts: ['a road with many cars', 'heavy traffic with multiple vehicles'] }, 63 { key: 'truck', label: 'Truck', keywords: ['truck', 'pickup'], 64 prompts: ['a photo of a truck', 'a large truck on the road'] }, 65 { key: 'dark', label: 'Dark scene', keywords: ['dark scene', 'nighttime', 'low light'], 66 prompts: ['a dark night scene', 'nighttime view'] } 67 ]; 68 69 // ---- IndexedDB persistence ------------------------------------------- 70 const IDB_DB = 'tcv-ai-search'; 71 const IDB_VERSION = 2; 72 async function idbOpen() { 73 return new Promise((resolve, reject) => { 74 const req = indexedDB.open(IDB_DB, IDB_VERSION); 75 req.onupgradeneeded = () => { 76 const db = req.result; 77 if (!db.objectStoreNames.contains('events')) db.createObjectStore('events'); 78 if (!db.objectStoreNames.contains('meta')) db.createObjectStore('meta'); 79 // drop legacy store if present 80 if (db.objectStoreNames.contains('embeddings')) db.deleteObjectStore('embeddings'); 81 }; 82 req.onsuccess = () => resolve(req.result); 83 req.onerror = () => reject(req.error); 84 }); 85 } 86 87 function currentFolderName() { 88 return window.app?.folderParser?.rootHandle?.name || null; 89 } 90 91 async function persistIndex() { 92 const folderName = currentFolderName(); 93 if (!folderName) return; 94 const modelKey = state.currentModelKey; 95 const prefix = `${folderName}:${modelKey}:`; 96 try { 97 const db = await idbOpen(); 98 const tx = db.transaction(['events', 'meta'], 'readwrite'); 99 const eventsStore = tx.objectStore('events'); 100 // Clear any previous records for this folder+model (handles re-indexing) 101 const range = IDBKeyRange.bound(prefix, prefix + '\uffff'); 102 await new Promise((resolve, reject) => { 103 const req = eventsStore.delete(range); 104 req.onsuccess = resolve; 105 req.onerror = () => reject(req.error); 106 }); 107 for (const [eventId, record] of state.indexed) { 108 eventsStore.put(record, prefix + eventId); 109 } 110 tx.objectStore('meta').put({ 111 folderName, modelKey, strategy: state.strategy, 112 savedAt: Date.now(), eventCount: state.indexed.size 113 }, `${folderName}:${modelKey}`); 114 await new Promise((res, rej) => { tx.oncomplete = res; tx.onerror = () => rej(tx.error); }); 115 log(`Persisted ${state.indexed.size} events to IndexedDB (${folderName}, ${modelKey})`); 116 } catch (e) { 117 log(`Failed to persist index: ${e.message}`, 'warn'); 118 } 119 } 120 121 async function restoreIndex() {
122 const folderName = currentFolderName(); 123 if (!folderName) return false; 124 const modelKey = state.currentModelKey; 125 const prefix = `${folderName}:${modelKey}:`; 126 try { 127 const db = await idbOpen(); 128 const tx = db.transaction('events', 'readonly'); 129 const store = tx.objectStore('events'); 130 const range = IDBKeyRange.bound(prefix, prefix + '\uffff'); 131 state.indexed.clear(); 132 await new Promise((resolve, reject) => { 133 const req = store.openCursor(range); 134 req.onsuccess = (e) => { 135 const cursor = e.target.result; 136 if (!cursor) return resolve(); 137 state.indexed.set(cursor.key.slice(prefix.length), cursor.value); 138 cursor.continue(); 139 }; 140 req.onerror = () => reject(req.error); 141 }); 142 if (state.indexed.size > 0) { 143 state.enabled = true; 144 log(`Restored ${state.indexed.size} indexed events from IndexedDB`); 145 emit('status', { state: 'ready', message: `Restored ${state.indexed.size} indexed events` }); 146 emit('ready'); 147 return true; 148 } 149 } catch (e) { 150 log(`Restore failed: ${e.message}`, 'warn'); 151 } 152 return false; 153 } 154 155 // ---- State ----------------------------------------------------------- 156 const state = { 157 enabled: false, 158 currentModelKey: 'base', 159 strategy: 'sparse', // default per product decision: fast first run 160 indexed: new Map(), // eventId -> { frames: [...], tags: [...] } 161 indexing: false, 162 indexProgress: { done: 0, total: 0 }, 163 visionModel: null, 164 textModel: null, 165 imageProcessor: null, 166 tokenizer: null, 167 neutralEmbedding: null, 168 categoryEmbeddings: null, 169 lastQuery: null, 170 lastResults: null, 171 lastTagFilter: null, 172 lastTagFallback: null, 173 // Records what we actually loaded â 'fp16' | 'q8' | 'fp32'. q8 174 // produces compressed score distributions, so the search path 175 // uses lower thresholds and a softer baseline subtraction. 176 dtype: null, 177 device: null 178 }; 179 180 // ---- Event dispatch helpers ------------------------------------------ 181 function emit(name, detail = {}) { 182 window.dispatchEvent(new CustomEvent('ai-search:' + name, { detail })); 183 } 184 function log(msg, level = 'info') { 185 const prefix = '[AISearch]'; 186 if (level === 'err') console.error(prefix, msg); 187 else if (level === 'warn') console.warn(prefix, msg); 188 else console.log(prefix, msg); 189 } 190 191 // ---- Utility math ---------------------------------------------------- 192 function l2Normalize(vec) { 193 let s = 0; for (let i = 0; i < vec.length; i++) s += vec[i] * vec[i]; 194 const n = Math.sqrt(s) || 1; 195 const out = new Float32Array(vec.length); 196 for (let i = 0; i < vec.length; i++) out[i] = vec[i] / n; 197 return out; 198 } 199 function cosine(a, b) { 200 let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; 201 return s; 202 } 203 function averageAndNormalize(vecs) { 204 if (vecs.length === 0) return null; 205 const dim = vecs[0].length; 206 const out = new Float32Array(dim); 207 for (const v of vecs) { for (let i = 0; i < dim; i++) out[i] += v[i]; } 208 for (let i = 0; i < dim; i++) out[i] /= vecs.length; 209 return l2Normalize(out); 210 } 211 212 // ---- Time-anchor helpers --------------------------------------------- 213 function parseClipStartTime(filename) { 214 const m = filename.match(/^(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})/); 215 if (!m) return null; 216 const [, y, mo, d, h, mi, s] = m; 217 return new Date(`${y}-${mo}-${d}T${h}:${mi}:${s}`).getTime(); 218 } 219 function findClipForTime(clips, targetAbs) { 220 if (!clips || clips.length === 0) return null; 221 const starts = clips.map(c => parseClipStartTime(c.fileName || c.name)); 222 for (let i = 0; i < clips.length; i++) { 223 const s = starts[i]; 224 if (s == null) continue; 225 const end = (i + 1 < starts.length && starts[i + 1] != null) ? starts[i + 1] : (s + 60500); 226 if (targetAbs >= s && targetAbs < end) { 227 const offsetSec = Math.max(0.1, Math.min(59.9, (targetAbs - s) / 1000)); 228 return { clip: clips[i], offsetSec }; 229 } 230 } 231 return null; 232 } 233 234 // ---- CLIP model load ------------------------------------------------- 235 async function ensureTransformers() { 236 if (window.__tcvTransformersAI) return window.__tcvTransformersAI; 237 // Load Transformers.js on demand via dynamic import 238 const tf = await import('https://cdn.jsdelivr.net/npm/@huggingface/[email protected]'); 239 window.__tcvTransformersAI = tf; 240 return tf; 241 } 242 async function ensureClipLoaded() { 243 const wanted = state.currentModelKey; 244 if (state.visionModel && state.textModel) return; 245 const { AutoProcessor, AutoTokenizer, CLIPVisionModelWithProjection, CLIPTextModelWithProjection, env } = await ensureTransformers(); 246 env.allowLocalModels = false;
247 env.useBrowserCache = true; 248 // Silence the informational "VerifyEachNodeIsAssignedToAnEp" and similar 249 // ORT warnings that get fired on every WebGPU model load. Level 3 = 250 // errors only; real failures (session create errors, etc.) still surface. 251 // We hit several keys because ORT Web's logger config is split across 252 // the global env, each backend, AND the JSEP (WebGPU bridge) which 253 // uses a separate path that the per-backend setting doesn't reach. 254 try { env.logLevel = 'error'; } catch (e) { /* ignore */ } 255 try { if (env.backends?.onnx) env.backends.onnx.logSeverityLevel = 3; } catch (e) { /* ignore */ } 256 try { if (env.backends?.onnx) env.backends.onnx.logLevel = 'error'; } catch (e) { /* ignore */ } 257 try { if (env.backends?.onnx?.wasm) env.backends.onnx.wasm.logSeverityLevel = 3; } catch (e) { /* ignore */ } 258 try { if (env.backends?.onnx?.webgpu) env.backends.onnx.webgpu.logSeverityLevel = 3; } catch (e) { /* ignore */ } 259 const modelInfo = CLIP_MODELS[wanted]; 260 261 // Pick the strongest device + dtype combo this machine actually 262 // supports. fp16 requires the WebGPU `shader-f16` feature, which 263 // many Intel iGPUs and older Nvidia/AMD drivers don't expose â 264 // unconditionally requesting fp16 there throws "The device 265 // (webgpu) does not support fp16." and the user sees indexing 266 // fail. Probe up-front and degrade gracefully: 267 // 1. WebGPU + fp16 â fastest, smallest model (preferred) 268 // 2. WebGPU + fp32 â works on any WebGPU device, ~2à larger 269 // model download, full quality 270 // 3. WASM + fp32 â universal fallback, much slower 271 // 272 // We deliberately do NOT use q8 quantization here. The Xenova 273 // CLIP q8 weights produced degenerate vision embeddings in 274 // testing (all images mapping to nearly the same point in 275 // embedding space â every query returns identical scores for 276 // every frame). fp32 on WebGPU is a much safer slow path. 277 let device = 'wasm'; 278 let dtype = 'fp32'; 279 if ('gpu' in navigator) { 280 try { 281 const adapter = await navigator.gpu.requestAdapter(); 282 if (adapter) { 283 device = 'webgpu'; 284 dtype = adapter.features?.has('shader-f16') ? 'fp16' : 'fp32'; 285 if (dtype === 'fp32') { 286 log(`WebGPU adapter doesn't expose shader-f16; using fp32 model (larger download, full quality)`); 287 } 288 } 289 } catch (e) { 290 log(`WebGPU adapter probe failed (${e?.message || e}); falling back to WASM`); 291 } 292 } 293 294 emit('status', { state: 'loading-model', message: `Loading ${modelInfo.name}â¦` }); 295 state.imageProcessor = await AutoProcessor.from_pretrained(modelInfo.id); 296 state.tokenizer = await AutoTokenizer.from_pretrained(modelInfo.id); 297 298 // Belt-and-suspenders: even with the probe, if the chosen combo 299 // fails (e.g., the q8 weights aren't available for the model), 300 // retry once on WASM fp32 before surfacing the error.
301 const loadModels = async (d, t) => { 302 state.visionModel = await CLIPVisionModelWithProjection.from_pretrained(modelInfo.id, { dtype: t, device: d }); 303 state.textModel = await CLIPTextModelWithProjection.from_pretrained(modelInfo.id, { dtype: t, device: d }); 304 }; 305 306 // ORT's WASM C++ logger writes `[W:onnxruntime:...VerifyEachNodeIsAssignedToAnEp]` 307 // and friends directly to console.warn during session creation. The env-level 308 // logSeverityLevel doesn't reach the JSEP path on every build, so the warning 309 // still leaks through and bloats our diagnostics ring buffer on every model 310 // load. Drop those specific messages by wrapping console.warn around the load, 311 // then restore. Anything not matching the ORT pattern passes through normally. 312 const ortNoisePattern = /^\d{4}-\d{2}-\d{2}.*\[W:onnxruntime:/; 313 const originalWarn = console.warn; 314 console.warn = function (...args) { 315 const first = typeof args[0] === 'string' ? args[0] : ''; 316 if (ortNoisePattern.test(first)) return; 317 return originalWarn.apply(console, args); 318 }; 319 320 try { 321 try { 322 await loadModels(device, dtype); 323 state.dtype = dtype; 324 state.device = device; 325 log(`${modelInfo.name} loaded on ${device} (${dtype})`); 326 } catch (e) { 327 const msg = e?.message || String(e); 328 console.warn(`[AISearch] Model load failed on ${device}/${dtype}: ${msg} â retrying on wasm/fp32`); 329 await loadModels('wasm', 'fp32'); 330 state.dtype = 'fp32'; 331 state.device = 'wasm'; 332 log(`${modelInfo.name} loaded on wasm (fp32) after ${device}/${dtype} fallback`); 333 } 334 } finally { 335 console.warn = originalWarn; 336 } 337 } 338 339 // ---- Dtype-aware scoring tuning --------------------------------- 340 // CLIP embeddings on q8 quantization have a compressed dynamic range â 341 // typical adjusted scores drop by ~50% vs fp16 even when the semantic 342 // ranking is identical. Without these adjustments, q8 users see "0 343 // confident, 0 possible, N hidden" on queries that fp16 users see 344 // perfectly fine matches for. The numbers below are calibrated to put 345 // q8 "possible" right around where weak-but-real matches land 346 // empirically; can be retuned once we have more telemetry from 347 // diverse hardware. The diagnostic log line below the threshold 348 // filter prints actual top scores per query so future tuning has 349 // data. 350 function getEffectiveThresholds() { 351 const base = CLIP_MODELS[state.currentModelKey].thresholds; 352 if (state.dtype === 'q8') { 353 return { confident: base.confident * 0.5, possible: base.possible * 0.5, tag: base.tag * 0.5 }; 354 } 355 return base; 356 } 357 function getEffectiveBaselineAlpha() { 358 // Softer subtraction on q8 â the neutral baseline shrinks 359 // proportionally with the query similarity, so the strong 0.8 360 // multiplier ends up over-correcting and leaves nothing above 361 // threshold. 362 return state.dtype === 'q8' ? 0.5 : 0.8; 363 } 364 365 // ---- Image/text embedding -------------------------------------------- 366 let _embedCanvas = null; 367 let _embedCtx = null; 368 async function embedImage(bitmap) { 369 const { RawImage } = await ensureTransformers(); 370 if (!_embedCanvas) { 371 _embedCanvas = document.createElement('canvas'); 372 // Transformers.js calls getImageData on this canvas to feed the 373 // model. willReadFrequently switches Chrome to a CPU-backed 374 // canvas, skipping the GPUâCPU readback per embed. ~10-15% 375 // faster on long indexing runs and silences the Chrome 376 // "Multiple readback operations" warning. 377 _embedCtx = _embedCanvas.getContext('2d', { willReadFrequently: true }); 378 } 379 _embedCanvas.width = bitmap.width; 380 _embedCanvas.height = bitmap.height; 381 _embedCtx.drawImage(bitmap, 0, 0); 382 const rawImage = await RawImage.fromCanvas(_embedCanvas); 383 const processed = await state.imageProcessor(rawImage); 384 const out = await state.visionModel({ pixel_values: processed.pixel_values }); 385 return l2Normalize(new Float32Array(out.image_embeds.data)); 386 } 387 async function embedText(query) { 388 const inputs = state.tokenizer(query, { padding: true, truncation: true }); 389 const out = await state.textModel(inputs); 390 return l2Normalize(new Float32Array(out.text_embeds.data)); 391 } 392 async function embedQueryEnsemble(query) { 393 const embs = await Promise.all(QUERY_TEMPLATES.map(t => embedText(t(query)))); 394 return averageAndNormalize(embs); 395 } 396 397 // ---- Category + baseline embeddings ---------------------------------- 398 async function ensureNeutralEmbedding() { 399 if (state.neutralEmbedding) return state.neutralEmbedding; 400 const embs = await Promise.all(NEUTRAL_PROMPTS.map(p => embedText(p))); 401 state.neutralEmbedding = averageAndNormalize(embs); 402 return state.neutralEmbedding; 403 } 404 async function ensureCategoryEmbeddings() { 405 if (state.categoryEmbeddings) return state.categoryEmbeddings; 406 const out = {}; 407 for (const cat of SCENE_CATEGORIES) { 408 const embs = await Promise.all(cat.prompts.map(p => embedText(p))); 409 out[cat.key] = averageAndNormalize(embs); 410 } 411 state.categoryEmbeddings = out; 412 return out; 413 } 414 async function computeFrameBaselines() { 415 const neutral = await ensureNeutralEmbedding(); 416 for (const record of state.indexed.values()) { 417 for (const f of record.frames) {
418 if (f.baseline != null) continue; 419 f.baseline = cosine(neutral, f.embedding); 420 } 421 } 422 } 423 async function computeEventTags() { 424 const cats = await ensureCategoryEmbeddings(); 425 await computeFrameBaselines(); 426 const minAdjusted = getEffectiveThresholds().tag; 427 for (const [eventId, record] of state.indexed) { 428 if (record.frames.length === 0) { record.tags = []; continue; } 429 const scores = {}; 430 for (const cat of SCENE_CATEGORIES) { 431 let best = -Infinity; 432 const catEmb = cats[cat.key]; 433 for (const f of record.frames) { 434 const adj = cosine(catEmb, f.embedding) - f.baseline; 435 if (adj > best) best = adj; 436 } 437 scores[cat.key] = best; 438 } 439 record.tags = Object.entries(scores) 440 .filter(([, s]) => s >= minAdjusted) 441 .sort(([, a], [, b]) => b - a) 442 .slice(0, 3) 443 .map(([key, score]) => ({ key, score, label: SCENE_CATEGORIES.find(c => c.key === key).label })); 444 } 445 } 446 447 // ---- Sampling planner ------------------------------------------------ 448 function planSamples(event, strategy = 'sparse') { 449 const samples = []; 450 const clipsByCamera = {}; 451 for (const clip of (event.clips || [])) { 452 (clipsByCamera[clip.camera] = clipsByCamera[clip.camera] || []).push(clip); 453 } 454 for (const cam of Object.keys(clipsByCamera)) { 455 clipsByCamera[cam].sort((a, b) => a.fileName.localeCompare(b.fileName)); 456 } 457 // Sentry: trigger-aware sampling. 458 // Default (sparse): just the 6 trigger-focused frames â Tesla's 459 // trigger IS the signal about what's important, so that's what we 460 // key off by default. 461 // Deep (motion/dense): keep the 6 trigger frames AND add motion- 462 // filtered sampling across the entire recording on the triggering 463 // camera, so the user can find unrelated things that happened 464 // outside the trigger window (e.g., something that was there 5 465 // minutes before the trigger fired). 466 const meta = event.metadata || event.metadata_event_json || null; 467 const triggerTime = meta?.timestamp ? new Date(meta.timestamp).getTime() : null; 468 const triggerCamRaw = meta?.camera != null ? String(meta.camera) : ''; 469 const triggerCam = CAMERA_ID_MAP[triggerCamRaw] || null; 470 if (event.type === 'SentryClips' && triggerTime && triggerCam && clipsByCamera[triggerCam]) { 471 // Trigger-focused samples â always added 472 for (const offset of SENTRY_OFFSETS) { 473 const targetAbs = triggerTime + offset * 1000; 474 const target = findClipForTime(clipsByCamera[triggerCam], targetAbs); 475 if (!target) continue; 476 samples.push({ 477 clip: target.clip, offsetInClip: target.offsetSec, absoluteTime: targetAbs, 478 camera: triggerCam, 479 tag: offset === 0 ? 'trigger' : `trigger${offset > 0 ? '+' : ''}${offset}s` 480 }); 481 } 482 if (clipsByCamera.front && triggerCam !== 'front') { 483 const ctx = findClipForTime(clipsByCamera.front, triggerTime); 484 if (ctx) samples.push({ 485 clip: ctx.clip, offsetInClip: ctx.offsetSec, absoluteTime: triggerTime, 486 camera: 'front', tag: 'front-ctx' 487 }); 488 } 489 // Deep-index: scan ALL cameras (not just the triggering one) with 490 // motion filtering. Someone walking past the repeater before the 491 // trigger fired was previously invisible to search. All cameras 492 // = full ~360° coverage around the car for the recording window. 493 if (strategy === 'motion' || strategy === 'dense') { 494 const allCams = ['front', 'back', 'left_repeater', 'right_repeater', 'left_pillar', 'right_pillar']; 495 for (const cam of allCams) { 496 const cams = clipsByCamera[cam]; 497 if (!cams || cams.length === 0) continue; 498 for (let i = 0; i < cams.length; i++) { 499 samples.push({ 500 clip: cams[i], 501 denseStrategy: strategy, 502 absoluteTime: parseClipStartTime(cams[i].fileName), 503 camera: cam, 504 tag: `${strategy}-${cam}-clip-${i + 1}/${cams.length}` 505 }); 506 } 507 } 508 } 509 return samples; 510 } 511 // SavedClips / RecentClips 512 const front = clipsByCamera.front || []; 513 if (strategy === 'dense' || strategy === 'motion') { 514 // Deep-index: scan ALL cameras, not just front. Catches anything 515 // on repeaters, rear cam, or pillars that front would miss. 516 const allCams = ['front', 'back', 'left_repeater', 'right_repeater', 'left_pillar', 'right_pillar']; 517 for (const cam of allCams) { 518 const cams = clipsByCamera[cam]; 519 if (!cams || cams.length === 0) continue; 520 for (let i = 0; i < cams.length; i++) { 521 samples.push({ 522 clip: cams[i], denseStrategy: strategy, 523 absoluteTime: parseClipStartTime(cams[i].fileName), 524 camera: cam, tag: `${strategy}-${cam}-clip-${i + 1}/${cams.length}` 525 }); 526 } 527 } 528 return samples; 529 } 530 // Sparse default â 1 frame per clip at 30s 531 for (let i = 0; i < front.length; i++) { 532 samples.push({ 533 clip: front[i], offsetInClip: 30, 534 absoluteTime: parseClipStartTime(front[i].fileName) + 30000,
535 camera: 'front', tag: `clip-${i + 1}/${front.length}` 536 }); 537 } 538 return samples; 539 } 540 541 // ---- Motion hash ----------------------------------------------------- 542 const _hashCanvas = (() => { const c = document.createElement('canvas'); c.width = 8; c.height = 8; return c; })(); 543 function computeHash(bitmap) { 544 const ctx = _hashCanvas.getContext('2d', { willReadFrequently: true }); 545 ctx.drawImage(bitmap, 0, 0, 8, 8); 546 const data = ctx.getImageData(0, 0, 8, 8).data; 547 const hash = new Uint8Array(192); 548 for (let i = 0; i < 64; i++) { hash[i * 3] = data[i * 4]; hash[i * 3 + 1] = data[i * 4 + 1]; hash[i * 3 + 2] = data[i * 4 + 2]; } 549 return hash; 550 } 551 function hashSimilar(a, b, threshold = 15) { 552 let diff = 0; 553 for (let i = 0; i < a.length; i++) diff += Math.abs(a[i] - b[i]); 554 return (diff / a.length) < threshold; 555 } 556 557 // ---- Thumbnail generation -------------------------------------------- 558 function makeThumb(bitmap) { 559 const TW = 240; 560 const TH = Math.round(TW * (bitmap.height / bitmap.width)); 561 const c = document.createElement('canvas'); 562 c.width = TW; c.height = TH; 563 c.getContext('2d').drawImage(bitmap, 0, 0, TW, TH); 564 return c.toDataURL('image/jpeg', 0.7); 565 } 566 567 // ---- Indexing --------------------------------------------------------- 568 async function indexEvent(event, strategy) { 569 const samples = planSamples(event, strategy); 570 if (samples.length === 0) return { frames: [], tags: [] }; 571 const byFile = new Map(); 572 for (const s of samples) { 573 const key = s.clip.fileName; 574 if (!byFile.has(key)) byFile.set(key, { clip: s.clip, samples: [] }); 575 byFile.get(key).samples.push(s); 576 } 577 const frames = []; 578 const motionState = { lastHash: null }; 579 for (const { clip, samples: fileSamples } of byFile.values()) { 580 const file = await clip.fileHandle.getFile(); 581 const decoder = new window.FastClipDecoder(); 582 try { 583 await decoder.init(file); 584 for (const samp of fileSamples) { 585 try { 586 if (samp.denseStrategy) { 587 const keyframeCtsList = decoder.samples.filter(s => s.is_sync).map(s => s.cts); 588 for (const cts of keyframeCtsList) { 589 const offsetSec = cts / decoder.timescale; 590 try { 591 const { bitmap, actualTime } = await decoder.extractAt(offsetSec); 592 let skip = false; 593 if (samp.denseStrategy === 'motion') { 594 const h = computeHash(bitmap); 595 if (motionState.lastHash && hashSimilar(h, motionState.lastHash)) skip = true; 596 else motionState.lastHash = h; 597 } 598 if (!skip) { 599 const embedding = await embedImage(bitmap); 600 frames.push({ 601 camera: samp.camera, 602 tag: `${samp.denseStrategy}-${actualTime.toFixed(1)}s`, 603 clipName: clip.fileName, 604 offsetInClip: actualTime, 605 thumbDataUrl: makeThumb(bitmap), 606 embedding 607 }); 608 } 609 bitmap.close?.(); 610 } catch (err) { 611 log(`frame extract failed: ${err.message}`, 'warn'); 612 } 613 } 614 } else { 615 const { bitmap, actualTime } = await decoder.extractAt(samp.offsetInClip); 616 const embedding = await embedImage(bitmap); 617 frames.push({ 618 camera: samp.camera, 619 tag: samp.tag, 620 clipName: clip.fileName, 621 offsetInClip: actualTime, 622 thumbDataUrl: makeThumb(bitmap), 623 embedding 624 }); 625 bitmap.close?.(); 626 } 627 } catch (err) { 628 log(`sample failed: ${err.message}`, 'warn'); 629 } 630 } 631 } finally { 632 decoder.close(); 633 } 634 } 635 return { frames, tags: [] }; 636 } 637 638 // Per-event time budget. SavedClips/Sentry events should take < 30s with 639 // WebCodecs. If we blow past, something's wrong with that event â 640 // abandon and move on rather than stalling the whole index. 641 const PER_EVENT_TIMEOUT_MS = 120000; 642 function withTimeout(promise, ms, label) { 643 return Promise.race([ 644 promise, 645 new Promise((_, rej) => setTimeout(() => rej(new Error(`${label || 'operation'} timed out after ${(ms/1000).toFixed(0)}s`)), ms)) 646 ]); 647 } 648 649 // Pause/cancel control flags 650 state.paused = false;
651 state.cancelled = false; 652 653 // Yield to the browser main thread so paints and input events have room 654 // to fire. Without this, the UI feels locked up during long indexes even 655 // though we're "awaiting" async calls â WebGPU/Canvas work is bursty on 656 // the main thread between awaits. 657 function yieldToMain() { return new Promise(r => setTimeout(r, 0)); } 658 659 async function indexAll(opts = {}) { 660 if (state.indexing) throw new Error('indexing already in progress'); 661 const allEvents = (opts.events || window.app?.eventBrowser?.events || []); 662 const events = allEvents.filter(e => e.clips?.length > 0 && e.type !== 'RecentClips'); 663 const skipped = allEvents.length - events.length; 664 if (events.length === 0) throw new Error('no indexable events (RecentClips are skipped)'); 665 const strategy = opts.strategy || state.strategy; 666 state.indexing = true; 667 state.paused = false; 668 state.cancelled = false; 669 state.indexProgress = { done: 0, total: events.length, currentName: '', startedAt: Date.now() }; 670 const skippedNote = skipped > 0 ? ` (skipping ${skipped} RecentClips)` : ''; 671 emit('status', { state: 'indexing', message: `Indexing ${events.length} events${skippedNote} (${strategy})â¦` }); 672 log(`Starting index: ${events.length} events, strategy=${strategy}${skippedNote}`); 673 try { 674 await ensureClipLoaded(); 675 for (let i = 0; i < events.length; i++) { 676 // Pause loop 677 while (state.paused && !state.cancelled) { 678 emit('progress', { done: i, total: events.length, status: '⸠Paused', paused: true }); 679 await new Promise(r => setTimeout(r, 400)); 680 } 681 if (state.cancelled) { log('Index cancelled by user', 'warn'); break; } 682 683 const ev = events[i]; 684 if (state.indexed.has(ev.name) && !opts.reindex) { 685 state.indexProgress.done++; 686 emit('progress', _progressDetail(i, events.length, `Skipped: ${ev.name}`, ev.name)); 687 continue; 688 } 689 state.indexProgress.currentName = ev.name; 690 emit('progress', _progressDetail(i, events.length, `Indexing ${ev.name}â¦`, ev.name)); 691 log(`[${i+1}/${events.length}] ${ev.name} (${ev.clips?.length || 0} clips)`); 692 try { 693 const record = await withTimeout(indexEvent(ev, strategy), PER_EVENT_TIMEOUT_MS, `indexEvent(${ev.name})`); 694 state.indexed.set(ev.name, record); 695 log(`[${i+1}/${events.length}] ${ev.name} â ${record.frames.length} frames`); 696 } catch (err) { 697 log(`[${i+1}/${events.length}] ${ev.name} FAILED: ${err.message}`, 'err'); 698 } 699 state.indexProgress.done = i + 1; 700 // Yield to main so the UI can paint progress + respond to input 701 await yieldToMain(); 702 } 703 if (!state.cancelled) { 704 emit('progress', _progressDetail(events.length, events.length, 'Computing tagsâ¦', '')); 705 await computeEventTags(); 706 state.enabled = true; 707 // Persist to IndexedDB so reloads don't force re-index 708 emit('progress', _progressDetail(events.length, events.length, 'Saving indexâ¦', '')); 709 await persistIndex(); 710 emit('status', { state: 'ready', message: `Indexed ${state.indexed.size} events (saved)` }); 711 emit('ready'); 712 } else { 713 // Still persist partial progress 714 await persistIndex(); 715 emit('status', { state: 'cancelled', message: `Indexing cancelled at ${state.indexed.size} events (partial saved)` }); 716 } 717 } finally { 718 state.indexing = false; 719 state.paused = false; 720 } 721 } 722 723 // Shape the progress event payload; includes ETA if we have enough signal 724 // to compute it (at least 2 events done). 725 function _progressDetail(done, total, status, currentName) {
726 const elapsed = Date.now() - (state.indexProgress.startedAt || Date.now()); 727 let etaMs = null; 728 if (done >= 2) { 729 const avgMs = elapsed / done; 730 etaMs = Math.max(0, avgMs * (total - done)); 731 } 732 return { done, total, status, currentName, etaMs, paused: state.paused }; 733 } 734 735 async function deepIndex(eventId) { 736 const ev = (window.app?.eventBrowser?.events || []).find(e => e.name === eventId); 737 if (!ev) throw new Error(`event not found: ${eventId}`); 738 await ensureClipLoaded(); 739 emit('status', { state: 'indexing', message: `Deep-indexing ${eventId}â¦` }); 740 state.indexed.delete(eventId); 741 const record = await indexEvent(ev, 'motion'); 742 state.indexed.set(eventId, record); 743 await computeEventTags(); 744 await persistIndex(); 745 emit('status', { state: 'ready', message: `Deep-indexed ${eventId} (saved)` }); 746 emit('ready'); 747 } 748 749 // ---- Query helpers ---------------------------------------------------- 750 function findCategoryFilterFromQuery(query) { 751 const q = query.toLowerCase(); 752 const all = []; 753 for (const cat of SCENE_CATEGORIES) for (const kw of (cat.keywords || [])) all.push({ kw, cat }); 754 all.sort((a, b) => b.kw.length - a.kw.length); 755 for (const { kw, cat } of all) if (q.includes(kw)) return cat; 756 return null; 757 } 758 759 async function search(query) { 760 if (!state.enabled) throw new Error('AI search not enabled'); 761 const q = (query || '').trim(); 762 if (!q) return { query: q, confident: [], possible: [], hiddenCount: 0 }; 763 // Lazy-load CLIP text model. We can index and then restore from IDB 764 // without loading CLIP, but searching needs the text encoder. 765 await ensureClipLoaded();
766 let tagFilter = findCategoryFilterFromQuery(q); 767 let tagFallback = null; 768 const te = await embedQueryEnsemble(q); 769 await computeFrameBaselines(); 770 const BASELINE_ALPHA = 0.8; 771 const eventById = new Map((window.app?.eventBrowser?.events || []).map(e => [e.name, e])); 772 // Score ALL indexed events against the query, then let tag-match be a 773 // ranking signal rather than a hard filter. Previously we pre-filtered 774 // to tagged-only events, which excluded deep-indexed events whose per- 775 // frame matches were strong but whose category score didn't quite hit 776 // the tag threshold. Now tag-match just boosts the score a bit. 777 const candidates = []; 778 for (const [id, rec] of state.indexed) { 779 const ev = eventById.get(id); 780 if (ev && rec.frames.length > 0) candidates.push({ id, rec, ev }); 781 } 782 const TAG_BOOST = 0.015; // small bump, not enough to override strong direct match 783 const effectiveAlpha = getEffectiveBaselineAlpha(); 784 const scored = candidates.map(({ id, rec, ev }) => { 785 let bestScore = -Infinity, bestFrame = null, bestRaw = 0; 786 for (const f of rec.frames) { 787 const raw = cosine(te, f.embedding); 788 const adj = raw - effectiveAlpha * (f.baseline ?? 0); 789 if (adj > bestScore) { bestScore = adj; bestFrame = f; bestRaw = raw; } 790 } 791 const hasTagMatch = tagFilter && (rec.tags || []).some(t => t.key === tagFilter.key); 792 const finalScore = hasTagMatch ? bestScore + TAG_BOOST : bestScore; 793 return { eventId: id, event: ev, tags: rec.tags || [], 794 score: finalScore, rawScore: bestRaw, bestFrame, 795 hasTagMatch }; 796 }).sort((a, b) => b.score - a.score); 797 798 // If a tag filter was detected but nothing scored above the possible 799 // threshold, note it as a fallback so the UI can show context 800 if (tagFilter && scored.filter(r => r.hasTagMatch).length === 0) { 801 tagFallback = tagFilter.label; 802 tagFilter = null; 803 } 804 805 const thresholds = getEffectiveThresholds(); 806 // Always use confidence tiers now that tag-filter is a boost, not a 807 // pre-filter. Deep-indexed events with strong query matches always 808 // show up regardless of whether their category tag was assigned. 809 const confident = scored.filter(r => r.score >= thresholds.confident); 810 const possible = scored.filter(r => r.score >= thresholds.possible && r.score < thresholds.confident); 811 const hiddenCount = scored.filter(r => r.score < thresholds.possible).length; 812 813 // Diagnostic: top-5 adjusted scores + corresponding raw cosines so we 814 // can tell q8-quantization compression apart from genuine-miss queries. 815 // Includes the effective threshold/alpha so the log is self-explanatory 816 // when comparing across dtypes (fp16 vs q8 vs fp32). 817 if (scored.length > 0) { 818 const top5 = scored.slice(0, 5).map(r => `${r.score.toFixed(3)}(raw ${r.rawScore.toFixed(3)})`); 819 console.log(`[AISearch] "${q}" dtype=${state.dtype} top-5: ${top5.join(', ')} | thresholds c=${thresholds.confident.toFixed(3)} p=${thresholds.possible.toFixed(3)} α=${effectiveAlpha}`); 820 } 821 822 state.lastQuery = q; 823 state.lastResults = { query: q, ranked: scored, confident, possible, hiddenCount, tagFilter, tagFallback }; 824 state.lastTagFilter = tagFilter; 825 state.lastTagFallback = tagFallback; 826 emit('results', state.lastResults); 827 return state.lastResults; 828 } 829 830 function getTags() { 831 const counts = {}; 832 for (const cat of SCENE_CATEGORIES) counts[cat.key] = 0; 833 for (const rec of state.indexed.values()) for (const t of (rec.tags || [])) counts[t.key] = (counts[t.key] || 0) + 1; 834 return SCENE_CATEGORIES.map(cat => ({ ...cat, count: counts[cat.key] || 0 })); 835 } 836 837 async function clearIndex() { 838 state.indexed.clear(); 839 state.enabled = false;
840 state.neutralEmbedding = null; 841 state.categoryEmbeddings = null; 842 // Drop the persisted copy for the current folder+model too 843 try { 844 const folderName = currentFolderName(); 845 if (folderName) { 846 const prefix = `${folderName}:${state.currentModelKey}:`; 847 const db = await idbOpen(); 848 const tx = db.transaction('events', 'readwrite'); 849 tx.objectStore('events').delete(IDBKeyRange.bound(prefix, prefix + '\uffff')); 850 } 851 } catch (e) { /* ignore */ } 852 emit('status', { state: 'cleared', message: 'Index cleared' }); 853 } 854 855 function getStatus() { 856 return { 857 enabled: state.enabled, 858 indexing: state.indexing, 859 progress: state.indexProgress, 860 eventCount: state.indexed.size, 861 frameCount: Array.from(state.indexed.values()).reduce((a, r) => a + r.frames.length, 0), 862 model: state.currentModelKey, 863 strategy: state.strategy, 864 webCodecs: !!(window.FastClipDecoder && window.FastClipDecoder.WEBCODECS_AVAILABLE) 865 }; 866 } 867 868 window.aiSearch = { 869 state, 870 enable: (opts) => indexAll({ ...opts, reindex: false }), 871 indexAll, 872 deepIndex, 873 search, 874 getTags, 875 clearIndex, 876 getStatus, 877 pause: () => { state.paused = true; }, 878 resume: () => { state.paused = false; }, 879 cancel: () => { state.cancelled = true; state.paused = false; }, 880 restoreIndex, 881 persistIndex, 882 SCENE_CATEGORIES, 883 CLIP_MODELS 884 }; 885})();
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.