https://videotext.io/assets/TranscriptionAccuracyBenchmark2026-CgokOcWt.js

1import{j as e}from"./react-vendor-mmlsrLMC.js";import{L as s}from"./router-DcHFLGSK.js";import{A as i}from"./AnswerBlock-BXAIzRjS.js";const o="/research/transcription-accuracy-benchmark-2026-pilot.csv",r={dataset:"LibriSpeech test-clean (public domain, professionally aligned reference transcripts)",condition:"Clean, single-speaker, read speech",model:'Whisper baseline — open-source "small" model (faster-whisper, int8, CPU)',utterances:25,speakers:5,audioMinutes:"4.3",corpusWer:"2.52%",realtimeFactor:"2.9x"},n=[{condition:"Clean speech (single speaker)",status:"Measured — Phase 1 pilot",detail:"25 LibriSpeech utterances, 5 speakers"},{condition:"Noisy / background-music speech",status:"Not yet measured",detail:"Requires a noise-augmented corpus (e.g. MUSAN-mixed speech)"},{condition:"Accented speech",status:"Not yet measured",detail:"Requires an accent-labeled corpus (e.g. Common Voice)"},{condition:"Multi-speaker conversation (diarization)",status:"Not yet measured",detail:"Requires speaker-labeled ground truth (e.g. AMI, CALLHOME)"},{condition:"Timestamp accuracy",status:"Not yet measured",detail:"Requires word-level forced-alignment ground truth"}],d=[{tool:"VideoText",status:"Not yet tested",note:"Requires running our production pipeline against the shared dataset"},{tool:"Whisper (open-source baseline)",status:"Tested — Phase 1",note:"2.52% corpus WER on clean speech (this pilot)"},{tool:"OpenAI Whisper API",status:"Not yet tested",note:"Requires an API key"},{tool:"Deepgram",status:"Not yet tested",note:"Requires an API key"},{tool:"AssemblyAI",status:"Not yet tested",note:"Requires an API key"},{tool:"YouTube auto-captions",status:"Not yet tested",note:"Requires captioned video sources"},{tool:"Otter.ai",status:"Not yet tested",note:"No public transcription API — requires manual export"},{tool:"Descript",status:"Not yet tested",note:"No public transcription API — requires manual export"}];function u(){return e.jsx("div",{className:"min-h-screen bg-white dark:bg-gray-950 py-12",children:e.jsxs("div",{className:"mx-auto max-w-5xl px-6 space-y-10",children:[e.jsx(s,{to:"/",className:"text-sm text-blue-600 hover:text-blue-700",children:"← Back to home"}),e.jsxs("div",{children:[e.jsx("p",{className:"text-xs font-semibold uppercase tracking-wide text-gray-500",children:"Research · Phase 1 pilot · Updated 2026"}),e.jsx("h1",{className:"mt-2 text-4xl font-medium text-gray-900 dark:text-white",children:"AI Transcription Accuracy Benchmark"}),e.jsx("p",{className:"mt-3 text-gray-600 dark:text-gray-300 max-w-3xl",children:"An open, reproducible study measuring transcription accuracy across tools using identical audio, ground-truth transcripts, and a public scoring methodology. This page publishes our dataset, our code, our results — including where we underperform — not a marketing comparison."})]}),e.jsx(i,{question:"How accurate is AI transcription today?",shortAnswer:`On clean, single-speaker speech, an open-source Whisper baseline scored ${r.corpusWer} word error rate in our Phase 1 pilot.`,expanded:"This is one data point from a small, reproducible pilot — not a finished 8-tool study. We're publishing it now, with full methodology and the underlying data, and will expand it condition-by-condition and tool-by-tool as described below. Treat every number on this page as scoped to exactly the condition and tool it's labeled with.",bullets:[`Dataset: ${r.dataset}`,`Condition tested so far: ${r.condition}`,`Sample: ${r.utterances} utterances, ${r.speakers} speakers, ${r.audioMinutes} minutes of audio`]}),e.jsxs("section",{className:"rounded-xl border border-amber-300 bg-amber-50/80 p-5 text-sm dark:border-amber-700 dark:bg-amber-950/30",children:[e.jsx("h2",{className:"text-lg font-medium text-gray-900 dark:text-white",children:"What this page is — and isn't — right now"}),e.jsxs("p",{className:"mt-2 text-gray-700 dark:text-gray-300",children:["This is a ",e.jsx("strong",{children:"Phase 1 pilot"}),`, not the full "8 tools × 200 files" benchmark. So far we've validated the scoring pipeline end-to-end on one clean-speech condition using an open-source Whisper model as a baseline — VideoText itself has not been run through this pipeline yet, and neither have Deepgram, AssemblyAI, Otter, Descript, or YouTube captions. We're publishing the pilot rather than waiting, because every number here is real, sourced, and reproducible, and we'd rather show partial honest progress than a complete but unverifiable comparison. See`," ",e.jsx("a",{href:"https://github.com/guntupalli09/videotext.io/tree/main/research/benchmark",className:"text-blue-700 hover:underline",target:"_blank",rel:"noreferrer",children:"the benchmark source code and README"})," ","for exactly what's measured and what isn't yet."]})]}),e.jsxs("section",{children:[e.jsx("h2",{className:"text-2xl font-medium text-gray-900 dark:text-white mb-4",children:"Phase 1 results (measured)"}),e.jsx("div",{className:"overflow-x-auto rounded-xl border border-gray-200 dark:border-gray-800",children:e.jsxs("table",{className:"min-w-full text-sm",children:[e.jsx("thead",{className:"bg-gray-50 dark:bg-gray-900",children:e.jsxs("tr",{children:[e.jsx("th",{className:"px-4 py-3 text-left",children:"Metric"}),e.jsx("th",{className:"px-4 py-3 text-left",children:"Value"})]})}),e.jsx("tbody",{children:[["Model tested",r.model],["Condition",r.condition],["Dataset",r.dataset],["Utterances / speakers",`${r.utterances} / ${r.speakers}`],["Audio tested",`${r.audioMinutes} minutes`],["Corpus word error rate (WER)",r.corpusWer],["Processing speed",`${r.realtimeFactor} realtime (CPU, no GPU)`]].map(([t,a])=>e.jsxs("tr",{className:"border-t border-gray-200 dark:border-gray-800",children:[e.jsx("td",{className:"px-4 py-3 font-medium text-gray-900 dark:text-white",children:t}),e.jsx("td",{className:"px-4 py-3 text-gray-700 dark:text-gray-300",children:a})]},t))})]})}),e.jsxs("p",{className:"mt-3 text-sm text-gray-600 dark:text-gray-400",children:[e.jsx("a",{href:o,className:"text-blue-700 hover:underline",download:!0,children:"Download the per-utterance results CSV"})," ","— every row includes the reference transcript, the hypothesis transcript, and the exact error counts used to compute WER."]})]}),e.jsxs("section",{children:[e.jsx("h2",{className:"text-2xl font-medium text-gray-900 dark:text-white mb-4",children:"What's tested vs. planned"}),e.jsx("p",{className:"text-sm text-gray-600 dark:text-gray-400 mb-3",children:"Conditions:"}),e.jsx("div",{className:"overflow-x-auto rounded-xl border border-gray-200 dark:border-gray-800 mb

1-6",children:e.jsxs("table",{className:"min-w-full text-sm",children:[e.jsx("thead",{className:"bg-gray-50 dark:bg-gray-900",children:e.jsxs("tr",{children:[e.jsx("th",{className:"px-4 py-3 text-left",children:"Condition"}),e.jsx("th",{className:"px-4 py-3 text-left",children:"Status"}),e.jsx("th",{className:"px-4 py-3 text-left",children:"Detail"})]})}),e.jsx("tbody",{children:n.map(t=>e.jsxs("tr",{className:"border-t border-gray-200 dark:border-gray-800",children:[e.jsx("td",{className:"px-4 py-3",children:t.condition}),e.jsx("td",{className:`px-4 py-3 ${t.status.startsWith("Measured")?"text-emerald-700 dark:text-emerald-400 font-medium":"text-gray-500"}`,children:t.status}),e.jsx("td",{className:"px-4 py-3 text-gray-600 dark:text-gray-400",children:t.detail})]},t.condition))})]})}),e.jsx("p",{className:"text-sm text-gray-600 dark:text-gray-400 mb-3",children:"Tools:"}),e.jsx("div",{className:"overflow-x-auto rounded-xl border border-gray-200 dark:border-gray-800",children:e.jsxs("table",{className:"min-w-full text-sm",children:[e.jsx("thead",{className:"bg-gray-50 dark:bg-gray-900",children:e.jsxs("tr",{children:[e.jsx("th",{className:"px-4 py-3 text-left",children:"Tool"}),e.jsx("th",{className:"px-4 py-3 text-left",children:"Status"}),e.jsx("th",{className:"px-4 py-3 text-left",children:"Note"})]})}),e.jsx("tbody",{children:d.map(t=>e.jsxs("tr",{className:"border-t border-gray-200 dark:border-gray-800",children:[e.jsx("td",{className:"px-4 py-3 font-medium",children:t.tool}),e.jsx("td",{className:`px-4 py-3 ${t.status.startsWith("Tested")?"text-emerald-700 dark:text-emerald-400 font-medium":"text-gray-500"}`,children:t.status}),e.jsx("td",{className:"px-4 py-3 text-gray-600 dark:text-gray-400",children:t.note})]},t.tool))})]})})]}),e.jsxs("section",{className:"rounded-xl border border-gray-200 dark:border-gray-800 p-5",children:[e.jsx("h2",{className:"text-xl font-medium text-gray-900 dark:text-white",children:"Methodology"}),e.jsxs("ul",{className:"mt-3 list-disc pl-5 text-sm text-gray-700 dark:text-gray-300 space-y-1.5",children:[e.jsxs("li",{children:[e.jsx("strong",{children:"Metric:"})," Word Error Rate (WER) — substitutions + deletions + insertions, divided by reference word count — computed via a real Levenshtein-alignment implementation, not estimated. Text is lowercased and stripped of punctuation before scoring (standard practice, since not every engine outputs punctuation)."]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Ground truth:"})," LibriSpeech test-clean reference transcripts, which are professionally aligned to public-domain LibriVox audiobook recordings."]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Reproducibility:"})," every script (dataset prep, transcription runner, scorer, unit tests) is public in the VideoText.io repository, alongside the raw per-utterance results."]}),e.jsxs("li",{children:[e.jsx("strong",{children:"Planned additions:"})," Diarization Error Rate (DER) for speaker attribution and timestamp mean-absolute-error are implemented and unit-tested in the scoring engine already, but have not been run yet — they need multi-speaker and forced-alignment ground truth we don't have wired up yet (see Limitations)."]})]})]}),e.jsxs("section",{className:"rounded-xl border border-gray-200 dark:border-gray-800 p-5",children:[e.jsx("h2",{className:"text-xl font-medium text-gray-900 dark:text-white",children:"Limitations"}),e.jsxs("ul",{className:"mt-3 list-disc pl-5 text-sm text-gray-700 dark:text-gray-300 space-y-1.5",children:[e.jsx("li",{children:"Small sample: 25 utterances, one condition, one model. Not statistically representative of real-world audio diversity."}),e.jsx("li",{children:"Read audiobook speech is easier to transcribe than spontaneous speech, meetings, or phone calls — this pilot's low WER should not be read as a general-purpose accuracy claim."}),e.jsx("li",{children:"VideoText has not yet been benchmarked here. When it is, we will publish the result regardless of how it compares to other tools, including any categories where VideoText performs worse."}),e.jsx("li",{children:"Diarization and timestamp accuracy are not yet measured against any tool."})]})]}),e.jsxs("section",{className:"rounded-xl border border-blue-200 bg-blue-50/60 p-5 text-sm dark:border-blue-900 dark:bg-blue-950/30",children:[e.jsx("p",{className:"font-semibold text-gray-900 dark:text-white mb-2",children:"Citation"}),e.jsxs("p",{className:"text-gray-700 dark:text-gray-300",children:['VideoText.io, "AI Transcription Accuracy Benchmark" (Phase 1 pilot), 2026.'," ",e.jsx("span",{className:"text-gray-500",children:"videotext.io/research/transcription-accuracy-benchmark-2026"})]})]}),e.jsxs("section",{className:"rounded-xl border border-gray-200 dark:border-gray-800 p-4 text-sm",children:[e.jsx("p",{className:"font-semibold text-gray-900 dark:text-white mb-2",children:"Related pages"}),e.jsxs("div",{className:"flex flex-wrap gap-3",children:[e.jsx(s,{to:"/best-transcription-tool",className:"text-blue-700 hover:underline",children:"Best transcription tool"}),e.jsx(s,{to:"/video-to-transcript",className:"text-blue-700 hover:underline",children:"Video to transcript"}),e.jsx(s,{to:"/ai-transcription-tools",className:"text-blue-700 hover:underline",children:"AI transcription tools"}),e.jsx(s,{to:"/transcription-statistics",className:"text-blue-700 hover:underline",children:"Industry transcription statistics"}),e.jsx(s,{to:"/glossary/word-error-rate",className:"text-blue-700 hover:underline",children:"What is word error rate?"})]})]})]})})}export{u as default};