1score_table = { 2 "-": { 3 "Model": "Human Performance*", 4 "Method": "-", 5 "Source": "https://arxiv.org/abs/2310.02255", 6 "Date": "2023-10-03", 7 "ALL": 60.3, 8 "FQA": 59.7, 9 "GPS": 48.4, 10 "MWP": 73.0, 11 "TQA": 63.2, 12 "VQA": 55.9, 13 "ALG": 50.9, 14 "ARI": 59.2, 15 "GEO": 51.4, 16 "LOG": 40.7, 17 "NUM": 53.8, 18 "SCI": 64.9, 19 "STA": 63.9 20 }, 21 "1": { 22 "Model": "OpenAI o1 \ud83e\udd47", 23 "Method": "LMM \ud83d\uddbc\ufe0f", 24 "Source": "https://openai.com/index/learning-to-reason-with-llms/", 25 "Date": "2024-09-12", 26 "ALL": 73.9, 27 "FQA": "-", 28 "GPS": "-", 29 "MWP": "-", 30 "TQA": "-", 31 "VQA": "-", 32 "ALG": "-", 33 "ARI": "-", 34 "GEO": "-", 35 "LOG": "-", 36 "NUM": "-", 37 "SCI": "-", 38 "STA": "-" 39 }, 40 "2": { 41 "Model": "Grok-2 \ud83e\udd48", 42 "Method": "LMM \ud83d\uddbc\ufe0f", 43 "Source": "https://x.ai/blog/grok-2", 44 "Date": "2024-08-13", 45 "ALL": 69.0, 46 "FQA": "-", 47 "GPS": "-", 48 "MWP": "-", 49 "TQA": "-", 50 "VQA": "-", 51 "ALG": "-", 52 "ARI": "-", 53 "GEO": "-", 54 "LOG": "-", 55 "NUM": "-", 56 "SCI": "-", 57 "STA": "-" 58 }, 59 "3": { 60 "Model": "Grok-2 mini \ud83e\udd49", 61 "Method": "LMM \ud83d\uddbc\ufe0f", 62 "Source": "https://x.ai/blog/grok-2", 63 "Date": "2024-08-13", 64 "ALL": 68.1, 65 "FQA": "-", 66 "GPS": "-", 67 "MWP": "-", 68 "TQA": "-", 69 "VQA": "-", 70 "ALG": "-", 71 "ARI": "-", 72 "GEO": "-", 73 "LOG": "-", 74 "NUM": "-", 75 "SCI": "-", 76 "STA": "-" 77 }, 78 "4": { 79 "Model": "Claude 3.5 Sonnet", 80 "Method": "LMM \ud83d\uddbc\ufe0f", 81 "Source": "https://www.anthropic.com/news/claude-3-5-sonnet", 82 "Date": "2024-06-20", 83 "ALL": 67.7, 84 "FQA": "-", 85 "GPS": "-", 86 "MWP": "-", 87 "TQA": "-", 88 "VQA": "-", 89 "ALG": "-", 90 "ARI": "-", 91 "GEO": "-", 92 "LOG": "-", 93 "NUM": "-", 94 "SCI": "-", 95 "STA": "-" 96 }, 97 "5": { 98 "Model": "LLaVA-OneVision", 99 "Method": "LMM \ud83d\uddbc\ufe0f", 100 "Source": "https://llava-vl.github.io/blog/2024-08-05-llava-onevision/", 101 "Date": "2024-08-06", 102 "ALL": 67.5, 103 "FQA": "-", 104 "GPS": "-", 105 "MWP": "-", 106 "TQA": "-", 107 "VQA": "-", 108 "ALG": "-", 109 "ARI": "-", 110 "GEO": "-", 111 "LOG": "-", 112 "NUM": "-", 113 "SCI": "-", 114 "STA": "-" 115 }, 116 "6": { 117 "Model": "Ovis1.6-Gemma2-9B", 118 "Method": "LMM \ud83d\uddbc\ufe0f", 119 "Source": "https://rank.opencompass.org.cn/leaderboard-multimodal/?m=REALTIME", 120 "Date": "2024-09-19", 121 "ALL": 67.2, 122 "FQA": "-", 123 "GPS": "-", 124 "MWP": "-", 125 "TQA": "-", 126 "VQA": "-", 127 "ALG": "-", 128 "ARI": "-", 129 "GEO": "-", 130 "LOG": "-", 131 "NUM": "-", 132 "SCI": "-", 133 "STA": "-" 134 }, 135 "7": { 136 "Model": "InternVL2-Pro", 137 "Method": "LMM \ud83d\uddbc\ufe0f", 138 "Source": "https://github.com/OpenGVLab/InternVL", 139 "Date": "2024-09-04", 140 "ALL": 66.8, 141 "FQA": 70.6, 142 "GPS": 65.4, 143 "MWP": 76.9, 144 "TQA": 71.5, 145 "VQA": 48.0, 146 "ALG": 66.5, 147 "ARI": 62.3, 148 "GEO": 63.6, 149 "LOG": 27.0, 150 "NUM": 40.3, 151 "SCI": 65.6, 152 "STA": 81.1 153 }, 154 "8": { 155 "Model": "TextGrad (GPT-4o)", 156 "Method": "LMM \ud83d\uddbc\ufe0f", 157 "Source": "https://github.com/zou-group/textgrad", 158 "Date": "2024-07-08", 159 "ALL": 66.1, 160 "FQA": "-", 161 "GPS": "-", 162 "MWP": "-", 163 "TQA": "-", 164 "VQA": "-", 165 "ALG": "-", 166 "ARI": "-", 167 "GEO": "-", 168 "LOG": "-", 169 "NUM": "-", 170 "SCI": "-", 171 "STA": "-" 172 }, 173 "9": { 174 "Model": "Gemini 1.5 Pro (May 2024)", 175 "Method": "LMM \ud83d\uddbc\ufe0f", 176 "Source": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf", 177 "Date": "2024-05-17", 178 "ALL": 63.9, 179 "FQA": "-", 180 "GPS": "-", 181 "MWP": "-", 182 "TQA": "-", 183 "VQA": "-", 184 "ALG": "-", 185 "ARI": "-", 186 "GEO": "-", 187 "LOG": "-", 188 "NUM": "-", 189 "SCI": "-", 190 "STA": "-" 191 }, 192 "10": { 193 "Model": "GPT-4o", 194 "Method": "LMM \ud83d\uddbc\ufe0f", 195 "Source": "https://openai.com/index/hello-gpt-4o/", 196 "Date": "2024-05-13", 197 "ALL": 63.8, 198 "FQA": "-", 199 "GPS": "-", 200 "MWP": "-", 201 "TQA": "-", 202 "VQA": "-", 203 "ALG": "-", 204 "ARI": "-", 205 "GEO": "-", 206 "LOG": "-", 207 "NUM": "-", 208 "SCI": "-", 209 "STA": "-" 210 }, 211 "11": { 212 "Model": "InternVL-Chat-V1.2-Plus", 213 "Method": "LMM \ud83d\uddbc\ufe0f", 214 "Source": "https://arxiv.org/abs/2312.14238", 215 "Date": "2024-02-22", 216 "ALL": 59.9, 217 "FQA": 51.7, 218 "GPS": 61.1, 219 "MWP": 79.6, 220 "TQA": 52.5, 221 "VQA": 57.0, 222 "ALG": 54.5, 223 "ARI": 63.2, 224 "GEO": 61.1, 225 "LOG": 16.2, 226 "NUM": 48.6, 227 "SCI": 55.7, 228 "STA": 60.8 229 }, 230 "12": { 231 "Model": "Gemini 1.5 Flash (May 2024)", 232 "Method": "LMM \ud83d\uddbc\ufe0f", 233 "Source": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf", 234 "Date": "2024-05-17", 235 "ALL": 58.4, 236 "FQA": "-", 237 "GPS": "-", 238 "MWP": "-", 239 "TQA": "-", 240 "VQA": "-", 241 "ALG": "-", 242 "ARI": "-", 243 "GEO": "-", 244 "LOG": "-", 245 "NUM": "-", 246 "SCI": "-", 247 "STA": "-" 248 }, 249 "13": { 250 "Model": "GPT-4T 2024-04-09", 251 "Method": "LMM \ud83d\uddbc\ufe0f", 252 "Source": "https://openai.com/index/hello-gpt-4o/", 253 "Date": "2024-05-13", 254 "ALL": 58.1, 255 "FQA": "-", 256 "GPS": "-", 257 "MWP": "-", 258 "TQA": "-", 259 "VQA": "-", 260 "ALG": "-", 261 "ARI": "-", 262 "GEO": "-", 263 "LOG": "-", 264 "NUM": "-", 265 "SCI": "-", 266 "STA": "-" 267 }, 268 "14": { 269 "Model": "Pixtral 12B", 270 "Method": "LMM \ud83d\uddbc\ufe0f", 271 "Source": "https://x.com/_philschmid/status/1833954941624615151", 272 "Date": "2024-09-11", 273 "ALL": 58.0, 274 "FQA": "-", 275 "GPS": "-", 276 "MWP": "-", 277 "TQA": "-", 278 "VQA": "-", 279 "ALG": "-", 280 "ARI": "-", 281 "GEO": "-", 282 "LOG": "-", 283 "NUM": "-", 284 "SCI": "-", 285 "STA": "-" 286 }, 287 "15": { 288 "Model": "InternLM-XComposer2-VL-7B", 289 "Method": "LMM \ud83d\uddbc\ufe0f", 290 "Source": "https://github.com/InternLM/InternLM-XComposer", 291 "Date": "2024-01-22", 292 "ALL": 57.6, 293 "FQA": 55.0, 294 "GPS": 63.0, 295 "MWP": 73.7, 296 "TQA": 56.3, 297 "VQA": 39.7, 298 "ALG": 56.6, 299 "ARI": 52.4, 300 "GEO": 62.3, 301 "LOG": 8.1, 302 "NUM": 42.4, 303 "SCI": 59.0, 304 "STA": 64.1 305 }, 306 "16": { 307 "Model": "Gemini 1.0 Ultra", 308 "Method": "LMM \ud83d\uddbc\ufe0f", 309 "Source": "https://arxiv.org/abs/2312.11805", 310 "Date": "2023-12-06", 311 "ALL": 53.0, 312 "FQA": 49.1, 313 "GPS": 56.2, 314 "MWP": 53.8, 315 "TQA": 69.0, 316 "VQA": 40.2, 317 "ALG": 58.4, 318 "ARI": 45.9, 319 "GEO": 55.6, 320 "LOG": 21.6, 321 "NUM": 38.9, 322 "SCI": 62.3, 323 "STA": 59.5 324 }, 325 "17": { 326 "Model": "Grok-1.5V", 327 "Method": "LMM \ud83d\uddbc\ufe0f", 328 "Source": "https://x.ai/blog/grok-1.5v", 329 "Date": "2024-04-12", 330 "ALL": 52.8, 331 "FQA": "-", 332 "GPS": "-", 333 "MWP": "-", 334 "TQA": "-", 335 "VQA": "-", 336 "ALG": "-", 337 "ARI": "-", 338 "GEO": "-", 339 "LOG": "-", 340 "NUM": "-", 341 "SCI": "-", 342 "STA": "-" 343 }, 344 "18": { 345 "Model": "Gemini 1.5 Pro (Feb 2024)", 346 "Method": "LMM \ud83d\uddbc\ufe0f", 347 "Source": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf", 348 "Date": "2024-02-15", 349 "ALL": 52.1, 350 "FQA": "-", 351 "GPS": "-", 352 "MWP": "-", 353 "TQA": "-", 354 "VQA": "-", 355 "ALG": "-", 356 "ARI": "-", 357 "GEO": "-", 358 "LOG": "-", 359 "NUM": "-", 360 "SCI": "-", 361 "STA": "-" 362 }, 363 "19": { 364 "Model": "Claude 3 Opus", 365 "Method": "LMM \ud83d\uddbc\ufe0f", 366 "Source": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc61885
3667627/Model_Card_Claude_3.pdf", 367 "Date": "2024-03-04", 368 "ALL": 50.5, 369 "FQA": "-", 370 "GPS": "-", 371 "MWP": "-", 372 "TQA": "-", 373 "VQA": "-", 374 "ALG": "-", 375 "ARI": "-", 376 "GEO": "-", 377 "LOG": "-", 378 "NUM": "-", 379 "SCI": "-", 380 "STA": "-" 381 }, 382 "20": { 383 "Model": "GPT-4V (Playground)", 384 "Method": "LMM \ud83d\uddbc\ufe0f", 385 "Source": "https://arxiv.org/abs/2310.02255", 386 "Date": "2023-10-15", 387 "ALL": 49.9, 388 "FQA": 43.1, 389 "GPS": 50.5, 390 "MWP": 57.5, 391 "TQA": 65.2, 392 "VQA": 38.0, 393 "ALG": 53.0, 394 "ARI": 49.0, 395 "GEO": 51.0, 396 "LOG": 21.6, 397 "NUM": 20.1, 398 "SCI": 63.1, 399 "STA": 55.8 400 }, 401 "21": { 402 "Model": "Claude 3 Sonnet", 403 "Method": "LMM \ud83d\uddbc\ufe0f", 404 "Source": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", 405 "Date": "2024-03-04", 406 "ALL": 47.9, 407 "FQA": "-", 408 "GPS": "-", 409 "MWP": "-", 410 "TQA": "-", 411 "VQA": "-", 412 "ALG": "-", 413 "ARI": "-", 414 "GEO": "-", 415 "LOG": "-", 416 "NUM": "-", 417 "SCI": "-", 418 "STA": "-" 419 }, 420 "22": { 421 "Model": "InternVL-Chat-V1.2", 422 "Method": "LMM \ud83d\uddbc\ufe0f", 423 "Source": "https://arxiv.org/abs/2312.14238", 424 "Date": "2024-02-22", 425 "ALL": 47.7, 426 "FQA": 50.9, 427 "GPS": 61.1, 428 "MWP": 30.6, 429 "TQA": 48.1, 430 "VQA": 44.7, 431 "ALG": 52.3, 432 "ARI": 36.5, 433 "GEO": 58.2, 434 "LOG": 18.9, 435 "NUM": 30.6, 436 "SCI": 54.9, 437 "STA": 51.8 438 }, 439 "23": { 440 "Model": "Math-LLaVA-13B", 441 "Method": "LMM \ud83d\uddbc\ufe0f", 442 "Source": "http://arxiv.org/abs/2406.17294", 443 "Date": "2024-06-25", 444 "ALL": 46.6, 445 "FQA": 37.2, 446 "GPS": 57.7, 447 "MWP": 56.5, 448 "TQA": 51.3, 449 "VQA": 33.5, 450 "ALG": 53.0, 451 "ARI": 40.2, 452 "GEO": 56.5, 453 "LOG": 16.2, 454 "NUM": 33.3, 455 "SCI": 49.2, 456 "STA": 43.9 457 }, 458 "24": { 459 "Model": "LLaVA-NeXT-34B", 460 "Method": "LMM \ud83d\uddbc\ufe0f", 461 "Source": "https://llava-vl.github.io/blog/2024-01-30-llava-1-6/", 462 "Date": "2024-01-30", 463 "ALL": 46.5, 464 "FQA": "-", 465 "GPS": "-", 466 "MWP": "-", 467 "TQA": "-", 468 "VQA": "-", 469 "ALG": "-", 470 "ARI": "-", 471 "GEO": "-", 472 "LOG": "-", 473 "NUM": "-", 474 "SCI": "-", 475 "STA": "-" 476 }, 477 "25": { 478 "Model": "Claude 3 Haiku", 479 "Method": "LMM \ud83d\uddbc\ufe0f", 480 "Source": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf", 481 "Date": "2024-03-04", 482 "ALL": 46.4, 483 "FQA": "-", 484 "GPS": "-", 485 "MWP": "-", 486 "TQA": "-", 487 "VQA": "-", 488 "ALG": "-", 489 "ARI": "-", 490 "GEO": "-", 491 "LOG": "-", 492 "NUM": "-", 493 "SCI": "-", 494 "STA": "-" 495 }, 496 "26": { 497 "Model": "Gemini 1.0 Pro", 498 "Method": "LMM \ud83d\uddbc\ufe0f", 499 "Source": "https://arxiv.org/abs/2312.11805", 500 "Date": "2023-12-06", 501 "ALL": 45.2, 502 "FQA": 47.6, 503 "GPS": 40.4, 504 "MWP": 39.2, 505 "TQA": 61.4, 506 "VQA": 39.1, 507 "ALG": 45.2, 508 "ARI": 38.8, 509 "GEO": 41.0, 510 "LOG": 10.8, 511 "NUM": 32.6, 512 "SCI": 54.9, 513 "STA": 56.8 514 }, 515 "27": { 516 "Model": "Phi-3-Vision-128K-In", 517 "Method": "LMM \ud83d\uddbc\ufe0f", 518 "Source": "https://huggingface.co/microsoft/Phi-3-vision-128k-instruct", 519 "Date": "2024-05-21", 520 "ALL": 44.5, 521 "FQA": "-", 522 "GPS": "-", 523 "MWP": "-", 524 "TQA": "-", 525 "VQA": "-", 526 "ALG": "-", 527 "ARI": "-", 528 "GEO": "-", 529 "LOG": "-", 530 "NUM": "-", 531 "SCI": "-", 532 "STA": "-" 533 }, 534 "28": { 535 "Model": "Phi-3.5-Vision 4.2B", 536 "Method": "LMM \ud83d\uddbc\ufe0f", 537 "Source": "https://arxiv.org/abs/2404.14219", 538 "Date": "2024-04-22", 539 "ALL": 43.9, 540 "FQA": "-", 541 "GPS": "-", 542 "MWP": "-", 543 "TQA": "-", 544 "VQA": "-", 545 "ALG": "-", 546 "ARI": "-", 547 "GEO": "-", 548 "LOG": "-", 549 "NUM": "-", 550 "SCI": "-", 551 "STA": "-" 552 }, 553 "29": { 554 "Model": "Qwen-VL-Plus", 555 "Method": "LMM \ud83d\uddbc\ufe0f", 556 "Source": "https://github.com/QwenLM/Qwen-VL", 557 "Date": "2023-12-21", 558 "ALL": 43.3, 559 "FQA": 54.6, 560 "GPS": 38.5, 561 "MWP": 31.2, 562 "TQA": 55.1, 563 "VQA": 34.1, 564 "ALG": 39.1, 565 "ARI": 32.0, 566 "GEO": 39.3, 567 "LOG": 18.9, 568 "NUM": 26.4, 569 "SCI": 59.0, 570 "STA": 56.1 571 }, 572 "30": { 573 "Model": "Mini-Gemini-HD (Hermes-2-Yi-34B)", 574 "Method": "LMM \ud83d\uddbc\ufe0f", 575 "Source": "https://arxiv.org/abs/2403.18814", 576 "Date": "2024-03-27", 577 "ALL": 43.3, 578 "FQA": "-", 579 "GPS": "-", 580 "MWP": "-", 581 "TQA": "-", 582 "VQA": "-", 583 "ALG": "-", 584 "ARI": "-", 585 "GEO": "-", 586 "LOG": "-", 587 "NUM": "-", 588 "SCI": "-", 589 "STA": "-" 590 }, 591 "31": { 592 "Model": "SPHINX-MoE", 593 "Method": "MoE \ud83e\udd16", 594 "Source": "https://github.com/Alpha-VLLM/LLaMA2-Accessory/tree/main/SPHINX", 595 "Date": "2024-01-12", 596 "ALL": 42.3, 597 "FQA": 49.8, 598 "GPS": 31.2, 599 "MWP": 42.5, 600 "TQA": 46.8, 601 "VQA": 39.7, 602 "ALG": 31.7, 603 "ARI": 41.6, 604 "GEO": 30.5, 605 "LOG": 16.2, 606 "NUM": 27.1, 607 "SCI": 50.8, 608 "STA": 50.8 609 }, 610 "32": { 611 "Model": "Mini-Gemini (Mixtral-8x7B)", 612 "Method": "LMM \ud83d\uddbc\ufe0f", 613 "Source": "https://arxiv.org/abs/2403.18814", 614 "Date": "2024-03-27", 615 "ALL": 41.8, 616 "FQA": "-", 617 "GPS": "-", 618 "MWP": "-", 619 "TQA": "-", 620 "VQA": "-", 621 "ALG": "-", 622 "ARI": "-", 623 "GEO": "-", 624 "LOG": "-", 625 "NUM": "-", 626 "SCI": "-", 627 "STA": "-" 628 }, 629 "33": { 630 "Model": "MM1-7B-MoE-Chat ", 631 "Method": "LMM \ud83d\uddbc\ufe0f", 632 "Source": "https://arxiv.org/abs/2403.09611", 633 "Date": "2024-03-14", 634 "ALL": 40.9, 635 "FQA": "-", 636 "GPS": "-", 637 "MWP": "-", 638 "TQA": "-", 639 "VQA": "-", 640 "ALG": "-", 641 "ARI": "-", 642 "GEO": "-", 643 "LOG": "-", 644 "NUM": "-", 645 "SCI": "-", 646 "STA": "-" 647 }, 648 "34": { 649 "Model": "MiniCPM-V-2 (2.8B)", 650 "Method": "LMM \ud83d\uddbc\ufe0f", 651 "Source": "https://github.com/OpenBMB/MiniCPM-V", 652 "Date": "2024-04-14", 653 "ALL": 40.6, 654 "FQA": 53.2, 655 "GPS": 26.0, 656 "MWP": 37.1, 657 "TQA": 44.3, 658 "VQA": 39.1, 659 "ALG": 28.5, 660 "ARI": 33.1, 661 "GEO": 28.0, 662 "LOG": 10.8, 663 "NUM": 39.6, 664 "SCI": 48.4, 665 "STA": 51.8 666 }, 667 "35": { 668 "Model": "MM1-30B-Chat", 669 "Method": "LMM \ud83d\uddbc\ufe0f", 670 "Source": "https://arxiv.org/abs/2403.09611", 671 "Date": "2024-03-14", 672 "ALL": 39.4, 673 "FQA": "-", 674 "GPS": "-", 675 "MWP": "-", 676 "TQA": "-", 677 "VQA": "-", 678 "ALG": "-", 679 "ARI": "-", 680 "GEO": "-", 681 "LOG": "-", 682 "NUM": "-", 683 "SCI": "-", 684 "STA": "-" 685 }, 686 "36": { 687 "Model": "SPHINX-Plus", 688 "Method": "MoE \ud83e\udd16", 689 "Source": "https://github.com/Alpha-VLLM/LLaMA2-Accessory/tree/main/SPHINX", 690 "Date": "2024-01-12", 691 "ALL": 36.8, 692 "FQA": "-", 693 "GPS": "-", 694 "MWP": "-", 695 "TQA": "-", 696 "VQA": "-", 697 "ALG": "-", 698 "ARI": "-", 699 "GEO": "-", 700 "LOG": "-", 701 "NUM": "-", 702 "SCI": "-", 703 "STA": "-" 704 }, 705 "37": { 706 "Model": "SPHINX (V2)", 707 "Method": "LMM \ud83d\uddbc\ufe0f", 708 "Source": "https://github.com/Alpha-VLLM/LLaMA2-Accessory/tree/main/SPHINX", 709 "Date": "2023-11-17", 710 "ALL": 36.7, 711 "FQA": 54.6, 712 "GPS": 16.4, 713 "MWP": 23.1, 714 "TQA": 41.8, 715 "VQA": 43.0, 716 "ALG": 20.6, 717 "ARI": 33.4, 718 "GEO": 17.6, 719 "LOG": 24.3, 720 "NUM": 21.5, 721 "SCI": 43.4, 722 "STA": 51.5 723 }, 724 "38": { 725 "Model": "MM1-7B-Chat ", 726 "Method": "LMM \ud83d\uddbc\ufe0f", 727 "Source": "https://arxiv.org/abs/2403.09611", 728 "Date": "2024-03-14", 729 "ALL": 35.9, 730 "FQA": "-", 731 "GPS": "-", 732 "MWP": "-", 733 "TQA": "-", 734 "VQA": "-", 735 "ALG": "-", 736 "ARI": "-", 737 "GEO": "-", 738 "LOG": "-", 739 "NUM": "-", 740 "SCI": "-", 741 "STA": "-" 742 }, 743 "39": { 744 "Model": "SPHINX-Intern2", 745 "Method": "MoE \ud83e\udd16", 746 "Source": "https://github.com/Alpha-VLLM/LLaMA2-Accessory/tree/main/SPHINX", 747 "Date": "2024-01-12", 748 "ALL": 35.5, 749 "FQA": "-", 750 "GPS": "-", 751 "MWP": "-", 752 "TQA": "-", 753 "VQA": "-", 754 "ALG": "-", 755 "ARI": "-", 756 "GEO": "-", 757 "LOG": "-", 758 "NUM": "-", 759 "SCI": "-", 760 "STA": "-" 761 }, 762 "40": { 763 "Model": "OmniLMM-12B", 764 "Method": "LMM \ud83d\uddbc\ufe0f", 765 "Source": "https://github.com/OpenBMB/OmniLMM", 766 "Date": "2024-02-01", 767 "ALL": 34.9, 768 "FQA": 45.0, 769 "GPS": 17.8, 770 "MWP": 26.9, 771 "TQA": 44.9, 772 "VQA": 39.1, 773 "ALG": 23.1, 774 "ARI": 32.3, 775 "GEO": 20.9, 776 "LOG": 18.9, 777 "NUM": 27.8, 778 "SCI": 45.9, 779 "STA": 44.2 780 }, 781 "41": { 782 "Model": "Multimodal Bard", 783 "Method": "LMM \ud83d\uddbc\ufe0f", 784 "Source": "https://arxiv.org/abs/2310.02255", 785 "Date": "2023-10-03", 786 "ALL": 34.8, 787 "FQA": 26.0, 788 "GPS": 47.1, 789 "MWP": 29.6, 790 "TQA": 48.7, 791 "VQA": 26.8, 792 "ALG": 46.5, 793 "ARI": 28.6, 794 "GEO": 47.8, 795 "LOG": 13.5, 796 "NUM": 14.9, 797 "SCI": 47.5, 798 "STA": 33.0 799 }, 800 "42": { 801 "Model": "LLaVA-NeXT-Vicuna-7B", 802 "Method": "LMM \ud83d\uddbc\ufe0f", 803 "Source": "https://llava-vl.github.io/blog/2024-01-30-llava-next/", 804 "Date": "2024-01-30", 805 "ALL": 34.6, 806 "FQA": "-", 807 "GPS": "-", 808 "MWP": "-", 809 "TQA": "-", 810 "VQA": "-", 811 "ALG": "-", 812 "ARI": "-", 813 "GEO": "-", 814 "LOG": "-", 815 "NUM": "-", 816 "SCI": "-", 817 "STA": "-" 818 }, 819 "43": { 820 "Model": "PoT GPT-4 (Caption+OCR)", 821 "Method": "Tool \ud83d\udee0\ufe0f", 822 "Source": "https://arxiv.org/abs/2310.02255", 823 "Date": "2023-10-03", 824 "ALL": 33.9, 825 "FQA": 30.1, 826 "GPS": 39.4, 827 "MWP": 30.6, 828 "TQA": 39.9, 829 "VQA": 31.3, 830 "ALG": 37.4, 831 "ARI": 31.7, 832 "GEO": 41.0, 833 "LOG": 18.9, 834 "NUM": 20.1, 835 "SCI": 44.3, 836 "STA": 37.9 837 }, 838 "44": { 839 "Model": "CoT Claude (Caption+OCR)", 840 "Method": "Tool \ud83d\udee0\ufe0f", 841 "Source": "https://arxiv.org/abs/2310.02255", 842 "Date": "2023-10-03", 843 "ALL": 33.2, 844 "FQA": 27.5, 845 "GPS": 29.3, 846 "MWP": 36.0, 847 "TQA": 49.4, 848 "VQA": 29.1, 849 "ALG": 31.0, 850 "ARI": 32.9, 851 "GEO": 31.0, 852 "LOG": 16.2, 853 "NUM": 17.4, 854 "SCI": 50.8, 855 "STA": 37.2 856 }, 857 "45": { 858 "Model": "CoT GPT4 (Caption+OCR)", 859 "Method": "Tool \ud83d\udee0\ufe0f", 860 "Source": "https://arxiv.org/abs/2310.02255", 861 "Date": "2023-10-03", 862 "ALL": 33.2, 863 "FQA": 27.9, 864 "GPS": 31.7, 865 "MWP": 31.2, 866 "TQA": 51.9, 867 "VQA": 28.5, 868 "ALG": 33.5, 869 "ARI": 30.9, 870 "GEO": 32.2, 871 "LOG": 13.5, 872 "NUM": 12.5, 873 "SCI": 58.2, 874 "STA": 37.9 875 }, 876 "46": { 877 "Model": "CoT ChatGPT (Caption+OCR)", 878 "Method": "Tool \ud83d\udee0\ufe0f", 879 "Source": "https://arxiv.org/abs/2310.02255", 880 "Date": "2023-10-03", 881 "ALL": 33.2, 882 "FQA": 26.0, 883 "GPS": 31.7, 884 "MWP": 35.5, 885 "TQA": 48.1, 886 "VQA": 30.2, 887 "ALG": 32.4, 888 "ARI": 32.3, 889 "GEO": 33.0, 890 "LOG": 16.2, 891 "NUM": 17.4, 892 "SCI": 54.9, 893 "STA": 36.2 894 }, 895 "47": { 896 "Model": "MM1-3B-MoE-Chat", 897 "Method": "LMM \ud83d\uddbc\ufe0f", 898 "Source": "https://arxiv.org/abs/2403.09611", 899 "Date": "2024-03-14", 900 "ALL": 32.6, 901 "FQA": "-", 902 "GPS": "-", 903 "MWP": "-", 904 "TQA": "-", 905 "VQA": "-", 906 "ALG": "-", 907 "ARI": "-", 908 "GEO": "-", 909 "LOG": "-", 910 "NUM": "-", 911 "SCI": "-", 912 "STA": "-" 913 }, 914 "48": { 915 "Model": "MM1-3B-Chat", 916 "Method": "LMM \ud83d\uddbc\ufe0f", 917 "Source": "https://arxiv.org/abs/2403.09611", 918 "Date": "2024-03-14", 919 "ALL": 32.0, 920 "FQA": "-", 921 "GPS": "-", 922 "MWP": "-", 923 "TQA": "-", 924 "VQA": "-", 925 "ALG": "-", 926 "ARI": "-", 927 "GEO": "-", 928 "LOG": "-", 929 "NUM": "-", 930 "SCI": "-", 931 "STA": "-" 932 }, 933 "49": {
934 "Model": "Gemini 1.0 Nano 2", 935 "Method": "LMM \ud83d\uddbc\ufe0f", 936 "Source": "https://arxiv.org/abs/2312.11805", 937 "Date": "2023-12-06", 938 "ALL": 30.6, 939 "FQA": 28.6, 940 "GPS": 23.6, 941 "MWP": 30.6, 942 "TQA": 41.8, 943 "VQA": 31.8, 944 "ALG": 27.1, 945 "ARI": 29.8, 946 "GEO": 26.8, 947 "LOG": 10.8, 948 "NUM": 20.8, 949 "SCI": 40.2, 950 "STA": 33.5 951 }, 952 "50": { 953 "Model": "LLaVA-1.5-13B", 954 "Method": "LMM \ud83d\uddbc\ufe0f", 955 "Source": "https://llava-vl.github.io/blog/2024-01-30-llava-1-6/", 956 "Date": "2024-01-30", 957 "ALL": 27.6, 958 "FQA": "-", 959 "GPS": "-", 960 "MWP": "-", 961 "TQA": "-", 962 "VQA": "-", 963 "ALG": "-", 964 "ARI": "-", 965 "GEO": "-", 966 "LOG": "-", 967 "NUM": "-", 968 "SCI": "-", 969 "STA": "-" 970 }, 971 "51": { 972 "Model": "SPHINX (V1)", 973 "Method": "LMM \ud83d\uddbc\ufe0f", 974 "Source": "https://github.com/Alpha-VLLM/LLaMA2-Accessory/tree/main/SPHINX", 975 "Date": "2023-11-09", 976 "ALL": 27.5, 977 "FQA": 23.4, 978 "GPS": 23.1, 979 "MWP": 21.5, 980 "TQA": 39.9, 981 "VQA": 34.1, 982 "ALG": 25.6, 983 "ARI": 28.1, 984 "GEO": 23.4, 985 "LOG": 16.2, 986 "NUM": 17.4, 987 "SCI": 40.2, 988 "STA": 23.6 989 }, 990 "52": { 991 "Model": "Gemini 1.0 Nano 1", 992 "Method": "LMM \ud83d\uddbc\ufe0f", 993 "Source": "https://arxiv.org/abs/2312.11805", 994 "Date": "2023-12-06", 995 "ALL": 27.3, 996 "FQA": 30.9, 997 "GPS": 21.6, 998 "MWP": 23.7, 999 "TQA": 29.1, 1000 "VQA": 30.7, 1001 "ALG": 23.8, 1002 "ARI": 25.5, 1003 "GEO": 21.3, 1004 "LOG": 13.5, 1005 "NUM": 20.8, 1006 "SCI": 27.9, 1007 "STA": 30.9 1008 }, 1009 "53": { 1010 "Model": "PoT ChatGPT (Caption+OCR)", 1011 "Method": "Tool \ud83d\udee0\ufe0f", 1012 "Source": "https://arxiv.org/abs/2310.02255", 1013 "Date": "2023-10-03", 1014 "ALL": 26.8, 1015 "FQA": 24.5, 1016 "GPS": 26.4, 1017 "MWP": 23.7, 1018 "TQA": 33.5, 1019 "VQA": 27.9, 1020 "ALG": 27.8, 1021 "ARI": 26.1, 1022 "GEO": 28.0, 1023 "LOG": 18.9, 1024 "NUM": 13.2, 1025 "SCI": 33.6, 1026 "STA": 29.9 1027 }, 1028 "54": { 1029 "Model": "SPHINX-Tiny", 1030 "Method": "MoE \ud83e\udd16", 1031 "Source": "https://github.com/Alpha-VLLM/LLaMA2-Accessory/tree/main/SPHINX", 1032 "Date": "2024-01-12", 1033 "ALL": 26.4, 1034 "FQA": "-", 1035 "GPS": "-", 1036 "MWP": "-", 1037 "TQA": "-", 1038 "VQA": "-", 1039 "ALG": "-", 1040 "ARI": "-", 1041 "GEO": "-", 1042 "LOG": "-", 1043 "NUM": "-", 1044 "SCI": "-", 1045 "STA": "-" 1046 }, 1047 "55": { 1048 "Model": "LLaVA (LLaMA-2-13B)", 1049 "Method": "LMM \ud83d\uddbc\ufe0f", 1050 "Source": "https://arxiv.org/abs/2310.02255", 1051 "Date": "2023-10-03", 1052 "ALL": 26.1, 1053 "FQA": 26.8, 1054 "GPS": 29.3, 1055 "MWP": 16.1, 1056 "TQA": 32.3, 1057 "VQA": 26.3, 1058 "ALG": 27.3, 1059 "ARI": 20.1, 1060 "GEO": 28.8, 1061 "LOG": 24.3, 1062 "NUM": 18.3, 1063 "SCI": 37.3, 1064 "STA": 25.1 1065 }, 1066 "56": { 1067 "Model": "InstructBLIP (Vicuna-7B)", 1068 "Method": "LMM \ud83d\uddbc\ufe0f", 1069 "Source": "https://arxiv.org/abs/2310.02255", 1070 "Date": "2023-10-03", 1071 "ALL": 25.3, 1072 "FQA": 23.1, 1073 "GPS": 20.7, 1074 "MWP": 18.3, 1075 "TQA": 32.3, 1076 "VQA": 35.2, 1077 "ALG": 21.8, 1078 "ARI": 27.1, 1079 "GEO": 20.7, 1080 "LOG": 18.9, 1081 "NUM": 20.4, 1082 "SCI": 33.0, 1083 "STA": 23.1 1084 }, 1085 "57": { 1086 "Model": "LLaVAR", 1087 "Method": "LMM \ud83d\uddbc\ufe0f", 1088 "Source": "https://arxiv.org/abs/2310.02255", 1089 "Date": "2023-10-03", 1090 "ALL": 25.2, 1091 "FQA": 21.9, 1092 "GPS": 25.0, 1093 "MWP": 16.7, 1094 "TQA": 34.8, 1095 "VQA": 30.7, 1096 "ALG": 24.2, 1097 "ARI": 22.1, 1098 "GEO": 23.0, 1099 "LOG": 13.5, 1100 "NUM": 15.3, 1101 "SCI": 42.6, 1102 "STA": 21.9 1103 }, 1104 "58": { 1105 "Model": "LLaMA-Adapter-V2 (7B)", 1106 "Method": "LMM \ud83d\uddbc\ufe0f", 1107 "Source": "https://arxiv.org/abs/2310.02255", 1108 "Date": "2023-10-03", 1109 "ALL": 23.9, 1110 "FQA": 21.2, 1111 "GPS": 25.5, 1112 "MWP": 11.3, 1113 "TQA": 32.3, 1114 "VQA": 31.8, 1115 "ALG": 26.3, 1116 "ARI": 20.4, 1117 "GEO": 24.3, 1118 "LOG": 24.3, 1119 "NUM": 13.9, 1120 "SCI": 29.5, 1121 "STA": 18.3 1122 }, 1123 "59": { 1124 "Model": "miniGPT4 (LLaMA-2-7B)", 1125 "Method": "LMM \ud83d\uddbc\ufe0f", 1126 "Source": "https://arxiv.org/abs/2310.02255", 1127 "Date": "2023-10-03", 1128 "ALL": 23.1, 1129 "FQA": 18.6, 1130 "GPS": 26.0, 1131 "MWP": 13.4, 1132 "TQA": 30.4, 1133 "VQA": 30.2, 1134 "ALG": 28.1, 1135 "ARI": 21.0, 1136 "GEO": 24.7, 1137 "LOG": 16.2, 1138 "NUM": 16.7, 1139 "SCI": 25.4, 1140 "STA": 17.9 1141 }, 1142 "60": { 1143 "Model": "mPLUG-Owl (LLaMA-7B)", 1144 "Method": "LMM \ud83d\uddbc\ufe0f", 1145 "Source": "https://arxiv.org/abs/2310.02255", 1146 "Date": "2023-10-03", 1147 "ALL": 22.2, 1148 "FQA": 22.7, 1149 "GPS": 23.6, 1150 "MWP": 10.2, 1151 "TQA": 27.2, 1152 "VQA": 27.9, 1153 "ALG": 23.6, 1154 "ARI": 19.2, 1155 "GEO": 23.9, 1156 "LOG": 13.5, 1157 "NUM": 12.7, 1158 "SCI": 26.3, 1159 "STA": 21.4 1160 }, 1161 "61": { 1162 "Model": "IDEFICS (9B-Instruct)", 1163 "Method": "LMM \ud83d\uddbc\ufe0f", 1164 "Source": "https://arxiv.org/abs/2310.02255", 1165 "Date": "2023-10-03", 1166 "ALL": 19.8, 1167 "FQA": 21.6, 1168 "GPS": 21.1, 1169 "MWP": 6.5, 1170 "TQA": 25.9, 1171 "VQA": 24.0, 1172 "ALG": 22.1, 1173 "ARI": 15.0, 1174 "GEO": 19.8, 1175 "LOG": 18.9, 1176 "NUM": 9.9, 1177 "SCI": 24.6, 1178 "STA": 18.1 1179 }, 1180 "62": { 1181 "Model": "Random Chance", 1182 "Method": "-", 1183 "Source": "https://arxiv.org/abs/2310.02255", 1184 "Date": "2023-10-03", 1185 "ALL": 17.9, 1186 "FQA": 15.5, 1187 "GPS": 24.1, 1188 "MWP": 4.5, 1189 "TQA": 23.4, 1190 "VQA": 24.3, 1191 "ALG": 25.8, 1192 "ARI": 13.8, 1193 "GEO": 22.7, 1194 "LOG": 13.4, 1195 "NUM": 8.8, 1196 "SCI": 15.8, 1197 "STA": 14.3 1198 } 1199}
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.