1// GENERATED by scripts/build_leaderboard.py â do not hand-edit. 2// Update: rerun the script against the internal run tree, review the 3// results/leaderboard-2026-09-29.json diff, commit both files together. 4window.EEBENCH_LEADERBOARD = { 5 "generated_at": "2026-09-29", 6 "version": "V1", 7 "corpus": "eebench-v1-core", 8 "questions": 13, 9 "reasoning_policy": "Reasoning configuration is selected per model.", 10 "note": "Generated by scripts/build_leaderboard.py \u2014 do not hand-edit.", 11 "models": [ 12 { 13 "question_scores_pct": [ 14 24.7, 15 25.0, 16 33.3, 17 66.7, 18 66.7, 19 75.0, 20 86.0, 21 97.3, 22 100.0, 23 100.0, 24 100.0, 25 100.0, 26 100.0 27 ], 28 "stderr_pct": 8.3, 29 "name": "Claude Opus 5.5", 30 "provider": "Anthropic", 31 "icon": "anthropic", 32 "scaffold": "Claude Code", 33 "reasoning_effort": "xhigh", 34 "score_pct": 75.0, 35 "questions_scored": 13, 36 "questions_total": 13, 37 "attempts": 39, 38 "avg_cost_usd": 2.366, 39 "total_cost_usd": 30.76, 40 "avg_output_tokens": 65077, 41 "avg_steps": 35.6, 42 "avg_duration_s": 802, 43 "cost_attempts": 39, 44 "cost_estimated": true, 45 "output_tokens_estimated": false 46 }, 47 { 48 "question_scores_pct": [ 49 0.0, 50 16.1, 51 19.5, 52 40.5, 53 50.0, 54 75.0, 55 100.0, 56 100.0, 57 100.0, 58 100.0, 59 100.0, 60 100.0, 61 100.0 62 ], 63 "stderr_pct": 10.7, 64 "name": "GPT-6 Astra", 65 "provider": "OpenAI", 66 "icon": "openai", 67 "scaffold": "Codex CLI", 68 "score_pct": 69.3, 69 "questions_scored": 13, 70 "questions_total": 13, 71 "attempts": 39, 72 "avg_cost_usd": 0.8295, 73 "total_cost_usd": 10.78, 74 "avg_output_tokens": 5967, 75 "avg_steps": 17.0, 76 "avg_duration_s": 106, 77 "cost_attempts": 39, 78 "cost_estimated": true, 79 "output_tokens_estimated": false 80 }, 81 { 82 "question_scores_pct": [ 83 8.3, 84 24.7, 85 25.0, 86 50.0, 87 66.7, 88 66.7, 89 68.8, 90 90.7, 91 94.7, 92 100.0, 93 100.0, 94 100.0, 95 100.0 96 ], 97 "stderr_pct": 9.1, 98 "name": "Claude Opus 5.5", 99 "provider": "Anthropic", 100 "icon": "anthropic", 101 "scaffold": "Claude Code", 102 "reasoning_effort": "high", 103 "score_pct": 68.9, 104 "questions_scored": 13, 105 "questions_total": 13, 106 "attempts": 39, 107 "avg_cost_usd": 0.6498, 108 "total_cost_usd": 8.45, 109 "avg_output_tokens": 20320, 110 "avg_steps": 16.4, 111 "avg_duration_s": 247, 112 "cost_attempts": 39, 113 "cost_estimated": true, 114 "output_tokens_estimated": false 115 }, 116 { 117 "question_scores_pct": [ 118 0.0, 119 0.0, 120 24.7, 121 25.0, 122 66.7, 123 76.7, 124 86.0, 125 95.2, 126 100.0, 127 100.0, 128 100.0, 129 100.0, 130 100.0 131 ], 132 "stderr_pct": 11.1, 133 "name": "Claude Sonnet 5.5", 134 "provider": "Anthropic", 135 "icon": "anthropic", 136 "scaffold": "Claude Code", 137 "reasoning_effort": "xhigh", 138 "score_pct": 67.2, 139 "questions_scored": 13, 140 "questions_total": 13, 141 "attempts": 39, 142 "avg_cost_usd": 0.8326, 143 "total_cost_usd": 10.82, 144 "avg_output_tokens": 44780, 145 "avg_steps": 24.8, 146 "avg_duration_s": 455, 147 "cost_attempts": 39, 148 "cost_estimated": true, 149 "output_tokens_estimated": false 150 }, 151 { 152 "question_scores_pct": [ 153 0.0, 154 25.0, 155 32.3, 156 33.3, 157 39.4, 158 58.3, 159 72.2, 160 85.3, 161 88.9, 162 97.3, 163 100.0, 164 100.0, 165 100.0 166 ], 167 "stderr_pct": 9.6, 168 "name": "Grok 4.7", 169 "provider": "xAI", 170 "icon": "xai", 171 "scaffold": "Grok CLI", 172 "score_pct": 64.0, 173 "questions_scored": 13, 174 "questions_total": 13, 175 "attempts": 39, 176 "avg_cost_usd": 3.5496, 177 "total_cost_usd": 46.14, 178 "avg_output_tokens": 81243, 179 "avg_steps": 54.7, 180 "avg_duration_s": 1234, 181 "cost_attempts": 39, 182 "cost_estimated": true, 183 "output_tokens_estimated": false 184 }, 185 { 186 "question_scores_pct": [ 187 0.0, 188 6.1, 189 18.3, 190 21.8, 191 24.7, 192 68.8, 193 92.0, 194 95.2, 195 100.0, 196 100.0, 197 100.0, 198 100.0, 199 100.0 200 ], 201 "stderr_pct": 11.6, 202 "name": "GPT-6.1 Sol", 203 "provider": "OpenAI", 204 "icon": "openai", 205 "scaffold": "Codex CLI", 206 "reasoning_effort": "high", 207 "score_pct": 63.6, 208 "questions_scored": 13, 209 "questions_total": 13, 210 "attempts": 39, 211 "avg_cost_usd": 0.3003, 212 "total_cost_usd": 3.9, 213 "avg_output_tokens": 17181, 214 "avg_steps": 27.1, 215 "avg_duration_s": 317, 216 "cost_attempts": 39, 217 "cost_estimated": true, 218 "output_tokens_estimated": false 219 }, 220 { 221 "question_scores_pct": [ 222 16.3, 223 28.1, 224 34.3, 225 62.5, 226 65.4, 227 67.5, 228 67.8, 229 68.5, 230 69.9, 231 70.9, 232 79.9, 233 80.7, 234 89.2 235 ], 236 "stderr_pct": 6.0, 237 "name": "Claude Opus 5", 238 "provider": "Anthropic", 239 "icon": "anthropic", 240 "scaffold": "Claude Code", 241 "score_pct": 61.6, 242 "questions_scored": 13, 243 "questions_total": 13, 244 "attempts": 39, 245 "avg_cost_usd": 2.3763, 246 "total_cost_usd": 30.89, 247 "avg_output_tokens": 49030, 248 "avg_steps": 26.5, 249 "avg_duration_s": 781, 250 "cost_attempts": 39 251 }, 252 { 253 "question_scores_pct": [ 254 0.0, 255 0.0, 256 22.1, 257 22.9, 258 40.0, 259 59.6, 260 72.7, 261 77.0, 262 78.3, 263 86.0, 264 91.7, 265 92.0, 266 100.0 267 ], 268 "stderr_pct": 9.9, 269 "name": "Grok 4.6", 270 "provider": "xAI", 271 "icon": "xai", 272 "scaffold": "Grok CLI", 273 "score_pct": 57.1, 274 "questions_scored": 13, 275 "questions_total": 13, 276 "attempts": 39, 277 "avg_cost_usd": 2.1109, 278 "total_cost_usd": 27.44, 279 "avg_output_tokens": 126635, 280 "avg_steps": 72.8, 281 "avg_duration_s": 2126, 282 "cost_attempts": 39, 283 "cost_estimated": true, 284 "output_tokens_estimated": false 285 }, 286 { 287 "question_scores_pct": [ 288 0.0, 289 18.3, 290 19.0, 291 25.0, 292 33.3, 293 43.8, 294 50.0,
295 61.0, 296 86.0, 297 97.3, 298 100.0, 299 100.0, 300 100.0 301 ], 302 "stderr_pct": 10.1, 303 "name": "Claude Fable 5.1", 304 "provider": "Anthropic", 305 "icon": "anthropic", 306 "scaffold": "Claude Code", 307 "reasoning_effort": "high", 308 "score_pct": 56.4, 309 "questions_scored": 13, 310 "questions_total": 13, 311 "attempts": 39, 312 "avg_cost_usd": 2.9159, 313 "total_cost_usd": 38.07, 314 "avg_output_tokens": 40465, 315 "avg_steps": 17.4, 316 "avg_duration_s": 652, 317 "cost_attempts": 38, 318 "cost_estimated": false, 319 "output_tokens_estimated": false 320 }, 321 { 322 "question_scores_pct": [ 323 0.0, 324 0.0, 325 0.0, 326 12.8, 327 25.0, 328 52.3, 329 75.0, 330 76.7, 331 95.2, 332 95.3, 333 100.0, 334 100.0, 335 100.0 336 ], 337 "stderr_pct": 11.9, 338 "name": "GPT-6 Sol", 339 "provider": "OpenAI", 340 "icon": "openai", 341 "scaffold": "Codex CLI", 342 "reasoning_effort": "xhigh", 343 "score_pct": 56.3, 344 "questions_scored": 13, 345 "questions_total": 13, 346 "attempts": 39, 347 "avg_cost_usd": 0.6487, 348 "total_cost_usd": 8.43, 349 "avg_output_tokens": 26759, 350 "avg_steps": 66.5, 351 "avg_duration_s": 359, 352 "cost_attempts": 39, 353 "cost_estimated": true, 354 "output_tokens_estimated": false 355 }, 356 { 357 "question_scores_pct": [ 358 0.0, 359 0.0, 360 5.6, 361 25.0, 362 40.0, 363 55.6, 364 66.7, 365 69.7, 366 77.8, 367 85.3, 368 94.7, 369 100.0, 370 100.0 371 ], 372 "stderr_pct": 10.4, 373 "name": "Gemini 3.8 Flash", 374 "provider": "Google", 375 "icon": "google", 376 "scaffold": "Gemini CLI", 377 "reasoning_effort": "high", 378 "score_pct": 55.4, 379 "questions_scored": 13, 380 "questions_total": 13, 381 "attempts": 39, 382 "avg_cost_usd": 1.3779, 383 "total_cost_usd": 17.91, 384 "avg_output_tokens": 68165, 385 "avg_steps": 96.7, 386 "avg_duration_s": 587, 387 "cost_attempts": 39, 388 "cost_estimated": false, 389 "output_tokens_estimated": false 390 }, 391 { 392 "question_scores_pct": [ 393 16.3, 394 16.7, 395 24.4, 396 30.8, 397 33.3, 398 51.8, 399 71.2, 400 71.3, 401 71.3, 402 72.3, 403 73.2, 404 79.9, 405 93.6 406 ], 407 "stderr_pct": 7.4, 408 "name": "Claude Fable 5", 409 "provider": "Anthropic", 410 "icon": "anthropic", 411 "scaffold": "Claude Code", 412 "score_pct": 54.3, 413 "questions_scored": 13, 414 "questions_total": 13, 415 "attempts": 39, 416 "avg_cost_usd": 3.1877, 417 "total_cost_usd": 41.44, 418 "avg_output_tokens": 33224, 419 "avg_steps": 27.3, 420 "avg_duration_s": 564, 421 "cost_attempts": 39 422 }, 423 { 424 "question_scores_pct": [ 425 16.3, 426 21.6, 427 23.8, 428 26.1, 429 36.6, 430 37.5, 431 46.3, 432 62.5, 433 69.6, 434 75.8, 435 78.5, 436 80.9, 437 92.0 438 ], 439 "stderr_pct": 7.3, 440 "name": "Claude Opus 4.8", 441 "provider": "Anthropic", 442 "icon": "anthropic", 443 "scaffold": "Claude Code", 444 "score_pct": 51.4, 445 "questions_scored": 13, 446 "questions_total": 13, 447 "attempts": 37, 448 "avg_cost_usd": 5.733, 449 "total_cost_usd": null, 450 "avg_output_tokens": 126821, 451 "avg_steps": 35.0, 452 "avg_duration_s": 2159, 453 "cost_attempts": 30, 454 "reasoning_effort": "max" 455 }, 456 { 457 "question_scores_pct": [ 458 0.0, 459 16.3, 460 17.5, 461 28.9, 462 50.4, 463 51.5, 464 52.1, 465 69.3, 466 71.3, 467 72.9, 468 73.0, 469 79.5, 470 79.6 471 ], 472 "stderr_pct": 7.5, 473 "name": "Grok 4.5", 474 "provider": "xAI", 475 "icon": "xai", 476 "scaffold": "Grok CLI", 477 "score_pct": 50.9, 478 "questions_scored": 13, 479 "questions_total": 13, 480 "attempts": 39, 481 "avg_cost_usd": 1.1427, 482 "total_cost_usd": 14.85, 483 "avg_output_tokens": 27806, 484 "avg_steps": 33.8, 485 "avg_duration_s": 372, 486 "cost_attempts": 39 487 }, 488 { 489 "question_scores_pct": [ 490 0.0, 491 9.3, 492 16.3, 493 26.3, 494 35.5, 495 42.0, 496 51.1, 497 52.1, 498 52.2, 499 60.6, 500 77.1, 501 79.7, 502 100.0 503 ], 504 "stderr_pct": 8.1, 505 "name": "Gemini 3.1 Pro", 506 "provider": "Google", 507 "icon": "google", 508 "scaffold": "Gemini CLI", 509 "score_pct": 46.3, 510 "questions_scored": 13, 511 "questions_total": 13, 512 "attempts": 39, 513 "avg_cost_usd": 0.8381, 514 "total_cost_usd": 10.95, 515 "avg_output_tokens": 28279, 516 "avg_steps": 28.4, 517 "avg_duration_s": 339, 518 "cost_attempts": 36 519 }, 520 { 521 "question_scores_pct": [ 522 0.0, 523 0.0, 524 0.0, 525 3.9, 526 14.2, 527 33.3, 528 40.3, 529 52.6, 530 70.2, 531 72.4, 532 79.7, 533 82.6, 534 100.0 535 ],
536 "stderr_pct": 10.1, 537 "name": "GPT-5.5", 538 "provider": "OpenAI", 539 "icon": "openai", 540 "scaffold": "API tools", 541 "score_pct": 42.3, 542 "questions_scored": 13, 543 "questions_total": 13, 544 "attempts": 41, 545 "avg_cost_usd": 3.1916, 546 "total_cost_usd": 40.88, 547 "avg_output_tokens": 19103, 548 "avg_steps": 26.7, 549 "avg_duration_s": 355, 550 "cost_attempts": 41 551 }, 552 { 553 "question_scores_pct": [ 554 0.0, 555 3.0, 556 16.2, 557 16.2, 558 24.4, 559 24.8, 560 31.5, 561 48.9, 562 55.4, 563 71.5, 564 75.5, 565 75.6, 566 80.9 567 ], 568 "stderr_pct": 8.1, 569 "name": "Claude Sonnet 5", 570 "provider": "Anthropic", 571 "icon": "anthropic", 572 "scaffold": "Claude Code", 573 "score_pct": 40.3, 574 "questions_scored": 13, 575 "questions_total": 13, 576 "attempts": 43, 577 "avg_cost_usd": 4.2053, 578 "total_cost_usd": 59.54, 579 "avg_output_tokens": 85208, 580 "avg_steps": 69.4, 581 "avg_duration_s": 1198, 582 "cost_attempts": 42 583 }, 584 { 585 "question_scores_pct": [ 586 0.0, 587 1.7, 588 3.9, 589 11.4, 590 13.0, 591 32.9, 592 36.7, 593 50.0, 594 61.7, 595 66.0, 596 72.8, 597 79.0, 598 83.6 599 ], 600 "stderr_pct": 8.7, 601 "name": "GPT-5.6 Sol", 602 "provider": "OpenAI", 603 "icon": "openai", 604 "scaffold": "API tools", 605 "score_pct": 39.4, 606 "questions_scored": 13, 607 "questions_total": 13, 608 "attempts": 39, 609 "avg_cost_usd": 1.6524, 610 "total_cost_usd": 21.48, 611 "avg_output_tokens": 11352, 612 "avg_steps": 17.7, 613 "avg_duration_s": 272, 614 "cost_attempts": 39 615 }, 616 { 617 "question_scores_pct": [ 618 0.0, 619 0.0, 620 23.6, 621 25.0, 622 26.9, 623 28.2, 624 39.3, 625 41.1, 626 46.3, 627 50.4, 628 67.5, 629 72.7, 630 79.3 631 ], 632 "stderr_pct": 6.9, 633 "name": "GPT-5.4", 634 "provider": "OpenAI", 635 "icon": "openai", 636 "scaffold": "API tools", 637 "score_pct": 38.5, 638 "questions_scored": 13, 639 "questions_total": 13, 640 "attempts": 38, 641 "avg_cost_usd": 2.6147, 642 "total_cost_usd": 35.61, 643 "avg_output_tokens": 37986, 644 "avg_steps": 30.4, 645 "avg_duration_s": 488, 646 "cost_attempts": 38 647 }, 648 { 649 "question_scores_pct": [ 650 0.0, 651 0.0, 652 0.0, 653 6.5, 654 14.6, 655 35.4, 656 63.4, 657 69.5, 658 71.9, 659 78.4, 660 81.6 661 ], 662 "stderr_pct": 10.5, 663 "name": "Kimi K3", 664 "provider": "Moonshot", 665 "icon": "kimi", 666 "scaffold": "OpenRouter", 667 "score_pct": 38.3, 668 "questions_scored": 11, 669 "questions_total": 13, 670 "attempts": 24, 671 "avg_cost_usd": 1.7948, 672 "total_cost_usd": 21.3, 673 "avg_output_tokens": 41096, 674 "avg_steps": 72.1, 675 "avg_duration_s": 1608, 676 "cost_attempts": 24 677 }, 678 { 679 "question_scores_pct": [ 680 0.0, 681 9.3, 682 9.8, 683 14.4, 684 16.3, 685 19.5, 686 27.9, 687 46.6, 688 52.1, 689 66.7, 690 70.1, 691 73.1, 692 80.2 693 ], 694 "stderr_pct": 7.9, 695 "name": "Gemini 3.6 Flash", 696 "provider": "Google", 697 "icon": "google", 698 "scaffold": "Gemini CLI", 699 "score_pct": 37.4, 700 "questions_scored": 13, 701 "questions_total": 13, 702 "attempts": 36, 703 "avg_cost_usd": 1.9177, 704 "total_cost_usd": 24.54, 705 "avg_output_tokens": 47303, 706 "avg_steps": 64.0, 707 "avg_duration_s": 797, 708 "cost_attempts": 36 709 }, 710 { 711 "question_scores_pct": [ 712 0.0, 713 9.1, 714 16.2, 715 17.8, 716 18.5, 717 19.5, 718 21.7, 719 33.3, 720 48.9, 721 51.7, 722 64.7, 723 74.8, 724 81.3 725 ], 726 "stderr_pct": 7.3, 727 "name": "Claude Opus 4.8", 728 "provider": "Anthropic", 729 "icon": "anthropic", 730 "scaffold": "Claude Code", 731 "score_pct": 35.2, 732 "questions_scored": 13, 733 "questions_total": 13, 734 "attempts": 46, 735 "avg_cost_usd": 1.4432, 736 "total_cost_usd": 18.23, 737 "avg_output_tokens": 34450, 738 "avg_steps": 19.2, 739 "avg_duration_s": 512, 740 "cost_attempts": 46, 741 "reasoning_effort": "high" 742 }, 743 { 744 "question_scores_pct": [ 745 0.0, 746 0.0, 747 13.0, 748 13.9, 749 14.4, 750 16.2, 751 18.2, 752 21.7, 753 49.1, 754 67.5, 755 67.5, 756 72.0, 757 91.9 758 ], 759 "stderr_pct": 8.6, 760 "name": "Gemini 3.5 Flash", 761 "provider": "Google", 762 "icon": "google", 763 "scaffold": "Gemini CLI", 764 "score_pct": 34.3, 765 "questions_scored": 13, 766 "questions_total": 13, 767 "attempts": 16, 768 "avg_cost_usd": 1.1122, 769 "total_cost_usd": 14.44, 770 "avg_output_tokens": 41102, 771 "avg_steps": 39.0, 772 "avg_duration_s": 452, 773 "cost_attempts": 15 774 }, 775 { 776 "question_scores_pct": [ 777 0.0, 778 0.0, 779 7.4, 780 13.0, 781 13.9, 782 16.3, 783 19.9, 784 24.9, 785 32.2, 786 52.1, 787 75.6, 788 79.5, 789 83.9 790 ], 791 "stderr_pct": 8.4, 792 "name": "GPT-5.6 Sol Pro", 793 "provider": "OpenAI", 794 "icon": "openai", 795 "scaffold": "OpenRouter", 796 "score_pct": 32.2, 797 "questions_scored": 13, 798 "questions_total": 13, 799 "attempts": 37, 800 "avg_cost_usd": 4.358, 801 "total_cost_usd": 56.83, 802 "avg_output_tokens": 85065, 803 "avg_steps": 21.6, 804 "avg_duration_s": 1617, 805 "cost_attempts": 37 806 }, 807 { 808 "question_scores_pct": [ 809 0.0, 810 0.0, 811 0.0, 812 0.0, 813 0.0, 814 18.8, 815 26.7, 816 33.3, 817 33.3, 818 58.3, 819 66.7, 820 66.7, 821 100.0 822 ], 823 "stderr_pct": 9.1, 824 "name": "GPT-6 Luna", 825 "provider": "OpenAI", 826 "icon": "openai", 827 "scaffold": "Codex CLI", 828 "reasoning_effort": "high", 829 "score_pct": 31.1, 830 "questions_scored": 13, 831 "questions_total": 13, 832 "attempts": 39, 833 "avg_cost_usd": 0.0368, 834 "total_cost_usd": 0.48, 835 "avg_output_tokens": 33945, 836 "avg_steps": 81.0, 837 "avg_duration_s": 388, 838 "cost_attempts": 39, 839 "cost_estimated": true, 840 "output_tokens_estimated": false 841 }, 842 { 843 "question_scores_pct": [ 844 0.0, 845 1.7, 846 3.9, 847 9.3, 848 12.2, 849 13.0, 850 16.3, 851 17.7, 852 46.2, 853 56.5, 854 64.6, 855 66.8, 856 92.8 857 ], 858 "stderr_pct": 8.5, 859 "name": "GPT-5.6 Terra", 860 "provider": "OpenAI", 861 "icon": "openai", 862 "scaffold": "API tools", 863 "score_pct": 30.8, 864 "questions_scored": 13, 865 "questions_total": 13, 866 "attempts": 39, 867 "avg_cost_usd": 0.8024, 868 "total_cost_usd": 10.43, 869 "avg_output_tokens": 11335, 870 "avg_steps": 17.6, 871 "avg_duration_s": 268, 872 "cost_attempts": 39 873 }, 874 { 875 "question_scores_pct": [ 876 0.0, 877 0.0, 878 0.0, 879 0.0, 880 0.0, 881 1.4, 882 11.8, 883 13.0, 884 16.3, 885 18.1, 886 19.5, 887 90.9, 888 94.2 889 ], 890 "stderr_pct": 9.1, 891 "name": "Claude Sonnet 4.6", 892 "provider": "Anthropic", 893 "icon": "anthropic", 894 "scaffold": "Claude Code", 895 "score_pct": 20.4, 896 "questions_scored": 13, 897 "questions_total": 13, 898 "attempts": 31, 899 "avg_cost_usd": 1.891, 900 "total_cost_usd": 30.39, 901 "avg_output_tokens": 80080, 902 "avg_steps": 29.7, 903 "avg_duration_s": 1439, 904 "cost_attempts": 26 905 }, 906 { 907 "question_scores_pct": [ 908 0.0, 909 0.0, 910 0.0, 911 0.0, 912 7.4, 913 11.8, 914 16.2, 915 21.7, 916 24.4, 917 24.7, 918 33.0, 919 40.4, 920 74.0 921 ], 922 "stderr_pct": 5.9, 923 "name": "Gemini 3 Flash", 924 "provider": "Google", 925 "icon": "google", 926 "scaffold": "Gemini CLI", 927 "score_pct": 19.5, 928 "questions_scored": 13, 929 "questions_total": 13, 930 "attempts": 24, 931 "avg_cost_usd": 0.4021, 932 "total_cost_usd": 5.75, 933 "avg_output_tokens": 53382, 934 "avg_steps": 33.8, 935 "avg_duration_s": 437, 936 "cost_attempts": 23 937 }, 938 { 939 "question_scores_pct": [ 940 0.0, 941 0.0, 942 0.0, 943 0.0, 944 0.0, 945 0.0, 946 5.4, 947 5.4, 948 6.5, 949 12.7, 950 44.2, 951 70.2, 952 94.1 953 ], 954 "stderr_pct": 8.6, 955 "name": "GPT-5.4 Mini", 956 "provider": "OpenAI", 957 "icon": "openai", 958 "scaffold": "API tools", 959 "score_pct": 18.3, 960 "questions_scored": 13, 961 "questions_total": 13, 962 "attempts": 46, 963 "avg_cost_usd": 2.0119, 964 "total_cost_usd": 26.52, 965 "avg_output_tokens": 146716, 966 "avg_steps": 48.7, 967 "avg_duration_s": 1140, 968 "cost_attempts": 37 969 }, 970 { 971 "question_scores_pct": [ 972 0.0, 973 0.0, 974 0.0, 975 0.0, 976 5.9, 977 8.1, 978 9.5, 979 14.6, 980 16.2, 981 21.7, 982 25.0, 983 50.1, 984 82.4 985 ], 986 "stderr_pct": 6.6, 987 "name": "Grok Build 0.1", 988 "provider": "xAI", 989 "icon": "xai", 990 "scaffold": "Grok CLI", 991 "score_pct": 18.0, 992 "questions_scored": 13, 993 "questions_total": 13, 994 "attempts": 28, 995 "avg_cost_usd": 1.6546, 996 "total_cost_usd": null, 997 "avg_output_tokens": 56810, 998 "avg_steps": 91.1, 999 "avg_duration_s": 669, 1000 "cost_attempts": 15 1001 }, 1002 { 1003 "question_scores_pct": [ 1004 0.0, 1005 0.0, 1006 4.1, 1007 4.5, 1008 5.0, 1009 8.2, 1010 9.3, 1011 14.0, 1012 19.6, 1013 29.3, 1014 30.7, 1015 33.8, 1016 58.7 1017 ], 1018 "stderr_pct": 4.8, 1019 "name": "GLM 5.2", 1020 "provider": "Z.ai", 1021 "icon": "glm", 1022 "scaffold": "OpenRouter", 1023 "score_pct": 16.7, 1024 "questions_scored": 13, 1025 "questions_total": 13, 1026 "attempts": 42, 1027 "avg_cost_usd": 0.5066, 1028 "total_cost_usd": 6.58, 1029 "avg_output_tokens": 43976, 1030 "avg_steps": 61.1, 1031 "avg_duration_s": 1114, 1032 "cost_attempts": 40, 1033 "reasoning_effort": "max" 1034 }, 1035 { 1036 "question_scores_pct": [ 1037 0.0, 1038 0.0, 1039 0.0, 1040 0.0, 1041 3.4, 1042 3.6, 1043 6.2, 1044 7.9, 1045 13.0, 1046 18.1, 1047 23.1, 1048 50.4, 1049 60.8 1050 ], 1051 "stderr_pct": 5.5, 1052 "name": "Grok 4.3", 1053 "provider": "xAI", 1054 "icon": "xai", 1055 "scaffold": "OpenRouter", 1056 "score_pct": 14.3, 1057 "questions_scored": 13, 1058 "questions_total": 13, 1059 "attempts": 39, 1060 "avg_cost_usd": 0.2298, 1061 "total_cost_usd": 2.99, 1062 "avg_output_tokens": 24711, 1063 "avg_steps": 31.6, 1064 "avg_duration_s": 327, 1065 "cost_attempts": 39 1066 }, 1067 { 1068 "question_scores_pct": [ 1069 0.0, 1070 0.0, 1071 0.0, 1072 0.0, 1073 1.3, 1074 6.5, 1075 8.1, 1076 8.3, 1077 11.8, 1078 29.4, 1079 31.8, 1080 33.6, 1081 35.4 1082 ], 1083 "stderr_pct": 4.0, 1084 "name": "Kimi K2.7 Code", 1085 "provider": "Moonshot", 1086 "icon": "kimi", 1087 "scaffold": "OpenRouter", 1088 "score_pct": 12.8, 1089 "questions_scored": 13, 1090 "questions_total": 13, 1091 "attempts": 40, 1092 "avg_cost_usd": 0.6384, 1093 "total_cost_usd": 8.49, 1094 "avg_output_tokens": 51742, 1095 "avg_steps": 62.6, 1096 "avg_duration_s": 1849, 1097 "cost_attempts": 36 1098 }, 1099 { 1100 "question_scores_pct": [ 1101 0.0, 1102 0.0, 1103 0.0, 1104 0.0, 1105 1.7, 1106 9.8, 1107 10.9, 1108 11.3, 1109 13.3, 1110 18.1, 1111 24.9, 1112 29.3, 1113 45.6 1114 ], 1115 "stderr_pct": 3.9, 1116 "name": "Qwen 3.6 Plus", 1117 "provider": "Alibaba", 1118 "icon": "qwen", 1119 "scaffold": "OpenRouter", 1120 "score_pct": 12.7, 1121 "questions_scored": 13, 1122 "questions_total": 13, 1123 "attempts": 40, 1124 "avg_cost_usd": 0.572, 1125 "total_cost_usd": 7.52, 1126 "avg_output_tokens": 28007, 1127 "avg_steps": 53.0, 1128 "avg_duration_s": 643, 1129 "cost_attempts": 40 1130 }, 1131 { 1132 "question_scores_pct": [ 1133 0.0, 1134 0.0, 1135 0.0, 1136 0.0, 1137 0.0, 1138 0.0, 1139 0.0, 1140 4.6, 1141 5.0, 1142 14.4, 1143 16.2, 1144 19.5, 1145 77.0 1146 ], 1147 "stderr_pct": 5.9, 1148 "name": "DeepSeek V4 Pro", 1149 "provider": "DeepSeek", 1150 "icon": "deepseek", 1151 "scaffold": "OpenRouter", 1152 "score_pct": 10.5, 1153 "questions_scored": 13, 1154 "questions_total": 13, 1155 "attempts": 33, 1156 "avg_cost_usd": 0.4028, 1157 "total_cost_usd": 5.91, 1158 "avg_output_tokens": 143279, 1159 "avg_steps": 44.8, 1160 "avg_duration_s": 2518, 1161 "cost_attempts": 29 1162 }, 1163 { 1164 "question_scores_pct": [ 1165 0.0, 1166 0.0, 1167 0.0, 1168 0.0, 1169 0.0, 1170 0.0, 1171 5.0, 1172 8.6, 1173 11.8, 1174 13.5, 1175 13.8, 1176 30.7, 1177 36.7 1178 ], 1179 "stderr_pct": 3.4, 1180 "name": "MiniMax M3", 1181 "provider": "MiniMax", 1182 "icon": "minimax", 1183 "scaffold": "OpenRouter", 1184 "score_pct": 9.2, 1185 "questions_scored": 13, 1186 "questions_total": 13, 1187 "attempts": 46, 1188 "avg_cost_usd": 0.2903, 1189 "total_cost_usd": 3.83, 1190 "avg_output_tokens": 97948, 1191 "avg_steps": 64.6, 1192 "avg_duration_s": 1301, 1193 "cost_attempts": 41 1194 }, 1195 { 1196 "question_scores_pct": [ 1197 0.0, 1198 0.0, 1199 0.0, 1200 0.0, 1201 0.0, 1202 0.0, 1203 0.0, 1204 5.4, 1205 6.7, 1206 9.3, 1207 12.1, 1208 14.4, 1209 29.3 1210 ], 1211 "stderr_pct": 2.4, 1212 "name": "Mistral Medium 3.5", 1213 "provider": "Mistral", 1214 "icon": "mistral", 1215 "scaffold": "OpenRouter", 1216 "score_pct": 5.9, 1217 "questions_scored": 13, 1218 "questions_total": 13, 1219 "attempts": 37, 1220 "avg_cost_usd": 5.2666, 1221 "total_cost_usd": 68.14, 1222 "avg_output_tokens": 70542, 1223 "avg_steps": 61.9, 1224 "avg_duration_s": 914, 1225 "cost_attempts": 37 1226 }, 1227 { 1228 "question_scores_pct": [ 1229 0.0, 1230 0.0, 1231 0.0, 1232 0.0, 1233 0.0, 1234 1.4, 1235 3.6, 1236 4.1, 1237 5.3, 1238 6.2, 1239 6.5, 1240 7.9, 1241 10.8 1242 ], 1243 "stderr_pct": 1.0, 1244 "name": "Claude Haiku 4.5", 1245 "provider": "Anthropic", 1246 "icon": "anthropic", 1247 "scaffold": "Claude Code", 1248 "score_pct": 3.5, 1249 "questions_scored": 13, 1250 "questions_total": 13, 1251 "attempts": 40, 1252 "avg_cost_usd": 0.412, 1253 "total_cost_usd": 5.32, 1254 "avg_output_tokens": 31601, 1255 "avg_steps": 53.6, 1256 "avg_duration_s": 344, 1257 "cost_attempts": 40 1258 } 1259 ] 1260};
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.