https://verificate.ai/_next/static/chunks/pages/fair-dinkum-8fb21a93c2f43e38.js

1(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[4093],{5713:(e,a,s)=>{"use strict";s.r(a),s.d(a,{default:()=>w});var t=s(7876),r=s(8047),n=s(8230),i=s.n(n),l=s(1298),o=s(8558),c=s(8300);let d={hidden:{opacity:0,y:18},show:function(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:0;return{opacity:1,y:0,transition:{duration:.5,delay:.06*e}}}},h="https://github.com/Verificate-Dev/fair-dinkum-bench",u=[],m=[{n:"23%",k:"of frontier-model answers on Australian law contained fabricated or absent case law (Allens AI Benchmark 2025)"},{n:"57.3",k:"point everyday-knowledge gap inside GPT-4 between US-proximate and far-market content (BLEnD, NeurIPS 2024)"},{n:"0.44",k:"Australian slang accuracy — below Indian English at 0.54 (Far Out benchmark 2026)"},{n:"911",k:"what US-trained models tell Australians to dial in an emergency. It's 000."}],p=[["Private eval set","~80% of questions are never published. Submissions are scored by maintainers; only scores are public."],["Rotating questions","The private set regenerates each release from a living Australian knowledge graph — fresh suburbs, donors, statutes. Memorising a leak buys one release."],["Time-locked truth","Statutory answers (wages, super caps, PBS prices, officeholders) are re-verified against primary sources on scoring day."],["Canary paraphrases","Unpublished variants of public items sit inside the private set. A public/private accuracy gap flags contamination."],["Transcript audits","Closed-book claims require full transcripts, spot-checked for tool-use fingerprints."],["Statistical rigor","95% bootstrap confidence intervals on every metric; ranks change only on a paired McNemar test at p<0.05."],["Third-party checkable","Ground truth re-verifiable against the ABS API by anyone: the v0.1 audit matched 19/19 (100%)."]],x=[{q:"What is Fair-Dinkum Bench?",a:"Fair-Dinkum Bench is the Australian LLM Challenge: 3,583 ground-truthed questions measuring whether an AI model actually knows Australia — its law, suburbs, money, institutions, culture and language. Every answer is sourced from authoritative Australian data (ABS, AEC, ABR, ACNC, the Federal Register of Legislation, AUSTLANG)."},{q:"What is the Fair-Dinkum Score (FDS)?",a:'Correct answers score +1, an honest "I don\'t know" scores 0, and a confidently wrong answer scores −1. A model that guesses can score below zero — worse than saying nothing. This inverts the guess-rewarding incentive of binary-accuracy benchmarks.'},{q:"How do I test my model on Fair-Dinkum Bench?",a:"Clone the GitHub repository, run the harness against any OpenAI-compatible endpoint (one temperature-0 call per question), and score with the open-source deterministic scorer. Submit results via pull request; maintainers verify against the private eval set and publish official numbers."},{q:"Why do frontier models score poorly on Australian questions?",a:"Documented research shows knowledge decays with distance from the US: fabricated Australian case law (23% of legal answers), US-defaulted emergency numbers and tax rules, sub-suburb facts answered with fluent invention, and annual statutory resets that outdate every parametric model each July 1."}],g={"@context":"https://schema.org","@type":"Dataset",name:"Fair-Dinkum Bench — The Australian LLM Challenge",alternateName:"Fair Dinkum Benchmark",description:"3,583 ground-truthed questions measuring LLM knowledge of Australia: law, demographics, socio-economics (SEIFA), political money, registers, culture, Indigenous-public data and language. Scored with the Fair-Dinkum Score, where honest abstention outranks confident error.",url:"".concat(c.W6,"/fair-dinkum"),sameAs:h,license:"https://creativecommons.org/licenses/by/4.0/",creator:{"@type":"Organization",name:"Verificate",url:c.W6},keywords:["Australian LLM benchmark","Fair-Dinkum Bench","Sovereign Australian AI","AI benchmark Australia","LLM leaderboard Australia","hallucination benchmark"],measurementTechnique:"Fair-Dinkum Score (correct +1, abstain 0, wrong −1), macro accuracy, Confident-Error Rate; 95% bootstrap CIs; McNemar-gated ranking"};function w(){return(0,t.jsxs)("div",{className:"bg-[#080A08] text-white/85 min-h-screen antialiased",children:[(0,t.jsx)(o.A,{title:"Fair-Dinkum Bench — The Australian LLM Challenge & Leaderboard",path:"/fair-dinkum",description:"The first benchmark measuring whether an AI actually knows Australia. 3,583 source-grounded questions across law, suburbs, money, culture and language. Honest 'I don't know' outranks confident wrongness — models can score below zero.",jsonLd:[g,(0,c.P$)(x)]}),(0,t.jsx)(l.Nv,{}),(0,t.jsx)(l.wn,{className:"pt-24 pb-10",children:(0,t.jsxs)(r.P.div,{initial:"hidden",animate:"show",variants:d,children:[(0,t.jsx)(l.YZ,{children:"THE AUSTRALIAN LLM CHALLENGE"}),(0,t.jsx)("h1",{className:"text-4xl md:text-6xl font-semibold tracking-tight text-white mt-3",children:"Fair-Dinkum Bench"}),(0,t.jsxs)("p",{className:"mt-5 max-w-3xl text-lg text-white/70",children:["Does your AI actually know Australia? 3,583 questions, every answer grounded in an authoritative Australian source. Fluent guessing is penalised: an honest"," ",(0,t.jsx)("span",{className:"text-white",children:"“I don’t know” outranks a wrong answer"}),", and a model can score ",(0,t.jsx)("span",{className:"text-white",children:"below zero"})," — worse than saying nothing."]}),(0,t.jsxs)("div",{className:"mt-8 grid grid-cols-2 md:grid-cols-4 gap-6",children:[(0,t.jsx)(l.ro,{n:"3,583",k:"ground-truthed questions"}),(0,t.jsx)(l.ro,{n:"14",k:"categories: law, suburbs, money, culture, language"}),(0,t.jsx)(l.ro,{n:"100%",k:"source-cited answers (ABS, AEC, ABR, Federal Register)"}),(0,t.jsx)(l.ro,{n:"−100 to +100",k:"Fair-Dinkum Score range"})]}),(0,t.jsxs)("div",{className:"mt-8 flex gap-4 flex-wrap",children:[(0,t.jsx)("a",{href:h,className:"rounded-xl bg-emerald-500 px-5 py-3 text-sm font-semibold text-black hover:bg-emerald-400",children:"Run it on your model →"}),(0,t.jsx)(l.cf,{href:"#leaderboard",children:"See the leaderboard"}),(0,t.jsx)(l.cf,{href:"/sovereign-australian-ai",children:"What is Sovereign Australian AI?"})]})]})}),(0,t.jsxs)(l.wn,{id:"score",className:"py-12",children:[(0,t.jsx)(l.YZ,{children:"THE FAIR-DINKUM SCORE"}),(0,t.jsx)("div",{className:"mt-6 grid md:grid-cols-3 gap-4",children:[["+1","Correct","Right answer, within the item's published tolerance, against the cited source."],["0","Honest abstention","“I don't know” is respectable. Real-world users are better served by honesty than fluent invention."],["−1","Confidently wrong","A wrong legal, health or emergency answer costs Australians. It costs models rank here."]].map((e,a)=>{let[s,n,i]=e;

1return(0,t.jsxs)(r.P.div,{custom:a,initial:"hidden",whileInView:"show",viewport:{once:!0},variants:d,className:"rounded-2xl border border-white/10 bg-white/[0.03] p-6",children:[(0,t.jsx)("div",{className:"text-4xl font-semibold text-emerald-400",children:s}),(0,t.jsx)("div",{className:"mt-2 text-white font-medium",children:n}),(0,t.jsx)("p",{className:"mt-2 text-sm text-white/60",children:i})]},n)})}),(0,t.jsx)("p",{className:"mt-4 text-sm text-white/50 max-w-3xl",children:"Binary-accuracy benchmarks teach models to guess — guessing beats abstaining on accuracy. Fair-Dinkum Bench inverts that incentive, and reports macro accuracy and Confident-Error Rate alongside FDS, all with 95% bootstrap confidence intervals."})]}),(0,t.jsxs)(l.wn,{id:"leaderboard",className:"py-12",children:[(0,t.jsx)(l.YZ,{children:"LEADERBOARD — CLOSED BOOK (model only, no tools)"}),(0,t.jsx)("div",{className:"mt-6 overflow-x-auto rounded-2xl border border-white/10",children:(0,t.jsxs)("table",{className:"w-full text-sm",children:[(0,t.jsx)("thead",{className:"bg-white/[0.04] text-white/60",children:(0,t.jsx)("tr",{children:["#","Model","FDS","Macro acc","Confident-Error Rate","Abstention","Run date"].map(e=>(0,t.jsx)("th",{className:"px-4 py-3 text-left font-medium",children:e},e))})}),(0,t.jsx)("tbody",{children:0===u.length?(0,t.jsx)("tr",{children:(0,t.jsx)("td",{colSpan:7,className:"px-4 py-8 text-center text-white/50",children:"Baseline sweep in progress — 10 models including GPT-5, Claude Opus 5, Gemini, Grok 4.5, Kimi K3 and Australian-hosted open models. Official numbers land here."})}):u.map((e,a)=>(0,t.jsxs)("tr",{className:"border-t border-white/5",children:[(0,t.jsx)("td",{className:"px-4 py-3 text-white/40",children:a+1}),(0,t.jsx)("td",{className:"px-4 py-3 text-white",children:e.model}),(0,t.jsx)("td",{className:"px-4 py-3 font-semibold text-emerald-400",children:e.fds}),(0,t.jsx)("td",{className:"px-4 py-3",children:e.acc}),(0,t.jsx)("td",{className:"px-4 py-3",children:e.cer}),(0,t.jsx)("td",{className:"px-4 py-3",children:e.abst}),(0,t.jsx)("td",{className:"px-4 py-3 text-white/50",children:e.date})]},e.model))})]})}),(0,t.jsx)("p",{className:"mt-3 text-xs text-white/40",children:"Ranks require statistical significance (paired McNemar, p<0.05) — otherwise reported as tied. Tools-allowed track (RAG, agents, knowledge graphs — citations required) opens with the public launch."})]}),(0,t.jsxs)(l.wn,{className:"py-12",children:[(0,t.jsx)(l.YZ,{children:"WHY THIS BENCHMARK EXISTS"}),(0,t.jsx)("div",{className:"mt-6 grid grid-cols-2 md:grid-cols-4 gap-6",children:m.map((e,a)=>(0,t.jsx)(r.P.div,{custom:a,initial:"hidden",whileInView:"show",viewport:{once:!0},variants:d,children:(0,t.jsx)(l.ro,{n:e.n,k:e.k})},e.n))})]}),(0,t.jsxs)(l.wn,{className:"py-12",children:[(0,t.jsx)(l.YZ,{children:"VERY HARD TO GAME — BY DESIGN"}),(0,t.jsx)("div",{className:"mt-6 grid md:grid-cols-2 lg:grid-cols-3 gap-4",children:p.map((e,a)=>{let[s,n]=e;return(0,t.jsxs)(r.P.div,{custom:a,initial:"hidden",whileInView:"show",viewport:{once:!0},variants:d,className:"rounded-2xl border border-white/10 bg-white/[0.03] p-5",children:[(0,t.jsx)("div",{className:"text-white font-medium",children:s}),(0,t.jsx)("p",{className:"mt-2 text-sm text-white/60",children:n})]},s)})})]}),(0,t.jsxs)(l.wn,{className:"py-12",children:[(0,t.jsx)(l.YZ,{children:"TEST YOUR MODEL IN TWO COMMANDS"}),(0,t.jsx)("pre",{className:"mt-6 rounded-2xl border border-white/10 bg-black/60 p-5 text-sm text-emerald-300/90 overflow-x-auto",children:"python harness/run_bench.py --base-url <any-openai-compatible-endpoint> \\\n    --model <your-model> --data data/public_dev_v0_1.jsonl --out results/run.jsonl\npython scoring/score.py results/run.jsonl   # FDS + macro accuracy + CER, with 95% CIs"}),(0,t.jsx)("p",{className:"mt-4 text-sm text-white/60 max-w-3xl",children:"One temperature-0 call per question. Submit via pull request with transcripts; maintainers verify and run the private eval set. Every ground truth carries its source dataset, record and vintage — and any Australian can challenge any item."}),(0,t.jsxs)("div",{className:"mt-6 flex gap-4 flex-wrap",children:[(0,t.jsx)("a",{href:h,className:"rounded-xl bg-emerald-500 px-5 py-3 text-sm font-semibold text-black hover:bg-emerald-400",children:"GitHub — data, scorer, harness"}),(0,t.jsx)(l.U5,{label:"Talk to us about Australian-grounded AI",secondary:!0})]})]}),(0,t.jsx)(l.wn,{className:"py-16 border-t border-white/10",children:(0,t.jsxs)("p",{className:"text-sm text-white/50 max-w-3xl",children:["Fair-Dinkum Bench is maintained by Verificate, builders of Kevin — the sovereign Australian knowledge graph. The graph that generates these questions is the same graph that answers them. That is the point: ",(0,t.jsx)(i(),{href:"/sovereign-australian-ai",className:"text-emerald-400 hover:underline",children:"Australians deserve an AI that knows Australia"}

1)," — and now there is a way to measure it."]})})]})}},8532:(e,a,s)=>{(window.__NEXT_P=window.__NEXT_P||[]).push(["/fair-dinkum",function(){return s(5713)}])}},e=>{e.O(0,[8096,8799,636,8792],()=>e(e.s=8532)),_N_E=e.O()}]);