1// image_search.js â "Search by image", fully client-side, logged-in users only. 2// 3// No image is ever sent to any server for object/text recognition: that 4// runs via TensorFlow.js + MobileNet, and Tesseract.js for OCR â both 5// WebAssembly/JS libraries executing entirely in the visitor's browser, lazy 6// loaded only when this modal is first opened. The shape-database matcher 7// (see below) only ever sends 7 small numbers describing an outline, never 8// the image itself. 9// 10// Flow: pick/drop/paste an image -> it's analyzed automatically -> on a 11// confident result, the page navigates straight to the search results. 12// There's no manual "type what this is" fallback â if nothing useful was 13// found, the modal asks for a different photo instead. 14 15var ImageSearch = (function () { 16 var mobilenetModel = null; 17 var tfReady = false; 18 var tesseractReady = false; 19 var initialized = false; 20 21 // ââ Lazy script loading âââââââââââââââââââââââââââââââââââââââââââââââââ 22 function loadScript(src) { 23 return new Promise(function (resolve, reject) { 24 if (document.querySelector('script[src="' + src + '"]')) { resolve(); return; } 25 var s = document.createElement('script'); 26 s.src = src; 27 s.onload = function () { resolve(); }; 28 s.onerror = function () { reject(new Error('Failed to load ' + src)); }; 29 document.head.appendChild(s); 30 }); 31 } 32 33 function ensureMobilenet() { 34 if (mobilenetModel) return Promise.resolve(); 35 var p = tfReady ? Promise.resolve() : loadScript('https://cdn.jsdelivr.net/npm/@tensorflow/[email protected]/dist/tf.min.js').then(function () { tfReady = true; }); 36 return p 37 .then(function () { return loadScript('https://cdn.jsdelivr.net/npm/@tensorflow-models/[email protected]/dist/mobilenet.min.js'); }) 38 .then(function () { 39 // Pin to a stable, direct Google Cloud Storage URL instead of the 40 // library's default loader. That default goes through tfhub.dev, 41 // which redirects to kaggle.com (Google migrated TF Hub model 42 // hosting there in late 2023) â a moving target that's already 43 // broken this twice. This GCS path has been stable for years and 44 // stays on a single domain already allowed in the CSP. 45 return mobilenet.load({ 46 version: 1, 47 alpha: 1.0, 48 modelUrl: 'https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_1.0_224/model.json', 49 }); 50 }) 51 .then(function (m) { mobilenetModel = m; }); 52 } 53 54 function ensureTesseract() { 55 if (tesseractReady) return Promise.resolve(); 56 return loadScript('https://cdn.jsdelivr.net/npm/tesseract.js@5/dist/tesseract.min.js') 57 .then(function () { tesseractReady = true; }); 58 } 59 60 function runOcr(imgEl) { 61 // 'eng' by default â add more language packs here if you need them, 62 // e.g. Tesseract.recognize(imgEl.src, 'eng+srp_latn') for Serbian Latin. 63 return Tesseract.recognize(imgEl.src, 'eng').then(function (res) { 64 return (res && res.data && res.data.text) ? res.data.text : ''; 65 }); 66 } 67 68 // ââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ 69 // MULTI-CROP CLASSIFICATION â test-time augmentation to help with photos 70 // that are already cropped/zoomed onto part of an object, or where the 71 // object only fills a corner of the frame. Classifies the full image AND 72 // two tighter center-crops, then keeps whichever run came back most 73 // confident. This is a standard CV technique, not a guess â it genuinely 74 // increases the odds that *one* of the crops frames the subject the way 75 // MobileNet expects, even if the original photo doesn't. 76 // Note: this only helps the MobileNet/object-recognition path. The shape 77 // database (Hu moments) below measures a complete outline by nature and 78 // can't be made robust to a cropped/cut-off silhouette â there's no fix 79 // for that without a different algorithm entirely. 80 // ââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ 81 function makeCenterCropCanvas(imgEl, fraction) { 82 try {
83 var iw = imgEl.naturalWidth || imgEl.width; 84 var ih = imgEl.naturalHeight || imgEl.height; 85 if (!iw || !ih) return null; 86 var cw = iw * fraction, ch = ih * fraction; 87 var cx = (iw - cw) / 2, cy = (ih - ch) / 2; 88 var canvas = document.createElement('canvas'); 89 canvas.width = 224; canvas.height = 224; // MobileNet's native input size 90 var ctx = canvas.getContext('2d'); 91 ctx.drawImage(imgEl, cx, cy, cw, ch, 0, 0, 224, 224); 92 return canvas; 93 } catch (e) { 94 return null; 95 } 96 } 97 98 function classifyMultiCrop(imgEl) { 99 var candidates = [imgEl, makeCenterCropCanvas(imgEl, 0.7), makeCenterCropCanvas(imgEl, 0.5)].filter(Boolean); 100 return Promise.all(candidates.map(function (el) { 101 return mobilenetModel.classify(el, 3).catch(function () { return []; }); 102 })).then(function (runs) { 103 var best = []; 104 for (var i = 0; i < runs.length; i++) { 105 if (runs[i].length && (!best.length || runs[i][0].probability > best[0].probability)) { 106 best = runs[i]; 107 } 108 } 109 return best; 110 }); 111 } 112 113 // ââ Combine recognition + OCR + image-database match into one query âââ 114 function buildQuery(labels, ocrText, imageMatch) { 115 var parts = []; 116 ocrText = (ocrText || '').replace(/\s+/g, ' ').trim(); 117 // OCR'd text is usually the more *specific* signal (a product name, a 118 // sign, a label) when it's short and clean â prioritize it. 119 if (ocrText && ocrText.length >= 2 && ocrText.length <= 80) parts.push(ocrText); 120 // Your own image database (embedding or shape match) is more specific 121 // to your catalog than a generic ImageNet label, so it outranks 122 // MobileNet's guess. 123 if (imageMatch && imageMatch.category) { 124 if (parts.indexOf(imageMatch.category) === -1) parts.push(imageMatch.category); 125 } else if (labels && labels.length && labels[0].probability > 0.10) { 126 var top = labels[0].className.split(',')[0].trim(); 127 if (top && parts.indexOf(top) === -1) parts.push(top); 128 } 129 return parts.join(' ').trim(); 130 } 131 132 function buildSummary(labels, ocrText, imageMatch) { 133 var bits = []; 134 if (imageMatch && imageMatch.category) { 135 bits.push((imageMatch.type === 'embedding' ? 'Matches your image database: ' : 'Matched your shape database: ') + imageMatch.category); 136 } 137 if (labels && labels.length) { 138 bits.push('Detected: ' + labels.slice(0, 3).map(function (l) { return l.className.split(',')[0]; }).join(', ')); 139 } 140 if (ocrText && ocrText.trim()) bits.push('text found in image'); 141 return bits.join(' · '); 142 } 143 144 // ââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ 145 // SHAPE MATCHING â color-invariant, built on your own server, no API. 146 // Computes 7 Hu moments (a standard shape descriptor â same technique 147 // OpenCV's cv2.HuMoments() uses) from the image's silhouette right here 148 // in the browser. Only those 7 numbers are sent to shape_match_api.php â 149 // never the image. This math MUST stay logically identical to the PHP 150 // version in shape_recognition_lib.php. 151 // ââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ 152 var SHAPE_SIZE = 64; 153 154 function computeHuMoments(imgEl) { 155 var canvas = document.createElement('canvas'); 156 canvas.width = SHAPE_SIZE; canvas.height = SHAPE_SIZE; 157 var ctx = canvas.getContext('2d'); 158 var data; 159 try { 160 ctx.drawImage(imgEl, 0, 0, SHAPE_SIZE, SHAPE_SIZE); 161 data = ctx.getImageData(0, 0, SHAPE_SIZE, SHAPE_SIZE).data; 162 } catch (e) { 163 return null; // canvas tainted â shouldn't happen with data: URLs, but be safe 164 } 165 166 var gray = new Array(SHAPE_SIZE * SHAPE_SIZE); 167 for (var i = 0; i < gray.length; i++) { 168 var r = data[i * 4], g = data[i * 4 + 1], b = data[i * 4 + 2]; 169 gray[i] = Math.round(0.299 * r + 0.587 * g + 0.114 * b); 170 } 171 return huMomentsFromGray(gray, SHAPE_SIZE, SHAPE_SIZE); 172 } 173 174 function otsuThreshold(gray, w, h) { 175 var hist = new Array(256).fill(0); 176 var total = w * h; 177 for (var i = 0; i < total; i++) hist[gray[i]]++; 178 var sumAll = 0; 179 for (var t = 0; t < 256;
179 t++) sumAll += t * hist[t]; 180 var sumB = 0, wB = 0, maxVar = 0, threshold = 127; 181 for (t = 0; t < 256; t++) { 182 wB += hist[t]; 183 if (wB === 0) continue; 184 var wF = total - wB; 185 if (wF === 0) break; 186 sumB += t * hist[t]; 187 var meanB = sumB / wB; 188 var meanF = (sumAll - sumB) / wF; 189 var variance = wB * wF * (meanB - meanF) * (meanB - meanF); 190 if (variance > maxVar) { maxVar = variance; threshold = t; } 191 } 192 return threshold; 193 } 194 195 function huMomentsFromGray(gray, w, h) { 196 var threshold = otsuThreshold(gray, w, h); 197 var corners = [gray[0], gray[w - 1], gray[(h - 1) * w], gray[(h - 1) * w + w - 1]]; 198 var cornerAvg = (corners[0] + corners[1] + corners[2] + corners[3]) / 4; 199 var bgIsLight = cornerAvg >= threshold; 200 201 var mask = new Array(w * h); 202 for (var i = 0; i < w * h; i++) { 203 var isLight = gray[i] >= threshold; 204 mask[i] = (isLight === bgIsLight) ? 0 : 1; 205 } 206 207 var M00 = 0, M10 = 0, M01 = 0; 208 for (var y = 0; y < h; y++) { 209 for (var x = 0; x < w; x++) { 210 if (!mask[y * w + x]) continue; 211 M00++; M10 += x; M01 += y; 212 } 213 } 214 if (M00 < 4) return null; 215 216 var xbar = M10 / M00, ybar = M01 / M00; 217 var mu20 = 0, mu02 = 0, mu11 = 0, mu30 = 0, mu03 = 0, mu12 = 0, mu21 = 0; 218 for (y = 0; y < h; y++) { 219 var dy = y - ybar; 220 for (x = 0; x < w; x++) { 221 if (!mask[y * w + x]) continue; 222 var dx = x - xbar; 223 mu20 += dx * dx; mu02 += dy * dy; mu11 += dx * dy; 224 mu30 += dx * dx * dx; mu03 += dy * dy * dy; 225 mu12 += dx * dy * dy; mu21 += dx * dx * dy; 226 } 227 } 228 229 function eta(mu, p, q) { return mu / Math.pow(M00, (p + q) / 2 + 1); } 230 var n20 = eta(mu20, 2, 0), n02 = eta(mu02, 0, 2), n11 = eta(mu11, 1, 1); 231 var n30 = eta(mu30, 3, 0), n03 = eta(mu03, 0, 3), n12 = eta(mu12, 1, 2), n21 = eta(mu21, 2, 1); 232 233 var t1 = n30 + n12, t2 = n21 + n03, t3 = n30 - 3 * n12, t4 = 3 * n21 - n03; 234 var I = []; 235 I[0] = n20 + n02; 236 I[1] = (n20 - n02) * (n20 - n02) + 4 * n11 * n11; 237 I[2] = t3 * t3 + t4 * t4; 238 I[3] = t1 * t1 + t2 * t2; 239 I[4] = t3 * t1 * (t1 * t1 - 3 * t2 * t2) + t4 * t2 * (3 * t1 * t1 - t2 * t2); 240 I[5] = (n20 - n02) * (t1 * t1 - t2 * t2) + 4 * n11 * t1 * t2; 241 I[6] = t4 * t1 * (t1 * t1 - 3 * t2 * t2) - t3 * t2 * (3 * t1 * t1 - t2 * t2); 242 243 return I.map(function (v) { 244 var sign = v < 0 ? -1 : 1; 245 return sign * Math.log10(Math.abs(v) + 1e-12); 246 }); 247 } 248 249 // Extracts a feature embedding from MobileNet's penultimate layer â much 250 // richer than the Hu moments above (captures texture/color pattern/learned 251 // visual semantics, not just outline). Needs the model already loaded. 252 function computeEmbedding(imgEl) { 253 try { 254 var t = mobilenetModel.infer(imgEl, true); 255 var arr = Array.prototype.slice.call(t.dataSync()); 256 t.dispose(); 257 return arr; 258 } catch (e) { 259 return null; 260 } 261 } 262 263 function matchImageOnServer(huVector, embedding) { 264 if (!huVector && !embedding) return Promise.resolve(null); 265 var fd = new URLSearchParams(); 266 if (huVector) fd.append('vector', JSON.stringify(huVector)); 267 if (embedding) fd.append('embedding', JSON.stringify(embedding)); 268 return fetch('shape_match_api.php', { method: 'POST', body: fd }) 269 .then(function (r) { return r.json(); }) 270 .then(function (d) { return (d.ok && d.match) ? d.match : null; }) 271 .catch(function () { return null; }); 272 } 273 274 function escHtml(s) { 275 return String(s).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>'); 276 } 277 278 // ââ UI state ââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ 279 function setStatus(text, showSpinner) { 280 var el = document.getElementById('imgSearchStatus'); 281 el.innerHTML = (showSpinner ? '<span class="imgsearch-spinner"></span>' : '') + escHtml(text); 282 } 283 284 // No manual fallback: a confident result auto-navigates to search; if 285 // nothing useful was found, the modal just asks for a different photo. 286 function showResult(query, labels, ocrText, imageMatch) { 287 if (query) { 288 setStatus(buildSummary(labels, ocrText, imageMatch) + ' â searchingâ¦'); 289 setTimeout(function () { 290 window.location.href = 'index.php?q=' + encodeURIComponent(query); 291 }, 650); 292 } else { 293 setStatus("Couldn't recognize anything specific in this image."); 294 document.getElementById('imgSearchRetryRow').style.display = 'block'; 295 } 296 } 297 298 function analyze(imgEl) { 299 // Hu-moment shape matching is local + fast (no CDN dependency). 300 var huVector = computeHuMoments(imgEl); 301 302 setStatus('Loading recognition models⦠(first time only, cached after)', true); 303 Promise.all([ensureMobilenet(), ensureTesseract()]) 304 .then(function () { 305 setStatus('Analyzing imageâ¦', true); 306 // The embedding needs the model loaded, so it's computed here 307 // rather than up front alongside the Hu vector. 308 var embedding = computeEmbedding(imgEl); 309 return Promise.all([ 310 classifyMultiCrop(imgEl), 311 runOcr(imgEl).catch(function () { return ''; }), 312 matchImageOnServer(huVector, embedding), 313 ]); 314 }) 315 .then(function (results) { 316 var labels = results[0], ocrText = results[1], imageMatch = results[2]; 317 showResult(buildQuery(labels, ocrText, imageMatch), labels, ocrText, imageMatch); 318 }) 319 .catch(function (err) { 320 console.error('ImageSearch analyze failed:', err); 321 // MobileNet/Tesseract failed (e.g. CDN unreachable) â no embedding 322 // is possible then either, but Hu-moment shape matching doesn't 323 // depend on any CDN and might still find something. 324 matchImageOnServer(huVector, null).then(function (imageMatch) { 325 showResult(buildQuery([], '', imageMatch), [], '', imageMatch); 326 }); 327 }); 328 } 329 330 function handleFile(file) { 331 if (!file || file.type.indexOf('image') === -1) return; 332 document.getElementById('imgSearchRetryRow').style.display = 'none'; 333 var reader = new FileReader(); 334 reader.onload = function (e) { 335 var img = document.getElementById('imgSearchPreview'); 336 img.onload = function () { analyze(img); }; 337 img.src = e.target.result; 338 img.style.display = 'block'; 339 document.getElementById('imgSearchDropInner').style.display = 'none'; 340 };
341 reader.readAsDataURL(file); 342 } 343 344 function reset() { 345 document.getElementById('imgSearchDropInner').style.display = 'flex'; 346 var img = document.getElementById('imgSearchPreview'); 347 img.style.display = 'none'; 348 img.src = ''; 349 document.getElementById('imgSearchFile').value = ''; 350 document.getElementById('imgSearchStatus').innerHTML = ''; 351 document.getElementById('imgSearchRetryRow').style.display = 'none'; 352 } 353 354 function retry() { 355 reset(); 356 } 357 358 function open() { 359 if (!window.IMG_SEARCH_LOGGED_IN) { 360 window.location.href = 'auth.php?mode=login&redirect=index.php'; 361 return; 362 } 363 init(); 364 reset(); 365 document.getElementById('imgSearchOverlay').classList.add('open'); 366 } 367 368 function close() { 369 document.getElementById('imgSearchOverlay').classList.remove('open'); 370 } 371 372 function init() { 373 if (initialized) return; 374 initialized = true; 375 376 var drop = document.getElementById('imgSearchDrop'); 377 drop.addEventListener('click', function (e) { 378 if (e.target.closest('.imgsearch-choose-btn') || e.target.closest('#imgSearchFile')) return; 379 if (document.getElementById('imgSearchPreview').style.display === 'block') return; // already showing a preview 380 document.getElementById('imgSearchFile').click(); 381 }); 382 drop.addEventListener('dragover', function (e) { e.preventDefault(); drop.classList.add('dragover'); }); 383 drop.addEventListener('dragleave', function () { drop.classList.remove('dragover'); }); 384 drop.addEventListener('drop', function (e) { 385 e.preventDefault(); 386 drop.classList.remove('dragover'); 387 if (e.dataTransfer.files && e.dataTransfer.files[0]) handleFile(e.dataTransfer.files[0]); 388 }); 389 390 document.addEventListener('paste', function (e) { 391 var overlay = document.getElementById('imgSearchOverlay'); 392 if (!overlay || !overlay.classList.contains('open')) return; 393 var items = (e.clipboardData || {}).items || []; 394 for (var i = 0; i < items.length; i++) { 395 if (items[i].type.indexOf('image') !== -1) { 396 handleFile(items[i].getAsFile()); 397 break; 398 } 399 } 400 }); 401 } 402 403 return { 404 open: open, 405 close: close, 406 handleFile: handleFile, 407 retry: retry, 408 }; 409})();
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.