[{"key":"gpqa","name":"GPQA","fullName":"Graduate-Level Google-Proof Q&A","category":"knowledge","description":"A challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. Designed to be difficult even for skilled non-experts with access to Google.","paperUrl":"https://arxiv.org/abs/2311.12022","paperTitle":"GPQA: A Graduate-Level Google-Proof Q&A Benchmark","authors":"David Rein, Betty Li Hou, Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, Samuel R. Bowman","year":"2023","tasks":"448 questions","format":"Multiple choice questions","decimals":null,"modelCount":117,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":96},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":94.6},{"slug":"claude-opus-4-7-adaptive","name":"Claude Opus 4.7 (Adaptive)","score":94.2},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":94.1},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":93.6}]},{"key":"sweVerified","name":"SWE-bench Verified","fullName":"Software Engineering Benchmark Verified","category":"coding","description":"SWE-bench Verified tests patches for 500 human-filtered GitHub issues. A score measures how many tasks an evaluated model-and-agent setup resolved; the model name alone does not identify that setup.","paperUrl":"https://arxiv.org/abs/2310.06770","paperTitle":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","authors":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","year":"2024","tasks":"500 verified issues","format":"Code patch generation","decimals":null,"modelCount":98,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":96},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":95.5},{"slug":"claude-fable","name":"Claude Fable 5","score":95},{"slug":"ember-1","name":"Ember-1","score":92.2},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":88.6}]},{"key":"designArenaWebsite","name":"Design Arena Website","fullName":"Design Arena Website Elo","category":"multimodalGrounded","description":"A display-only Design Arena website-generation Elo score surfaced on OpenRouter model benchmark pages.","paperUrl":"https://openrouter.ai/x-ai/grok-4.3/benchmarks","paperTitle":"OpenRouter Grok 4.3 benchmarks","authors":"Design Arena","year":"2026","tasks":"Website generation comparisons","format":"Elo","decimals":0,"modelCount":91,"lowerIsBetter":false,"top":[{"slug":"muse-wonder","name":"Muse Wonder","score":1360},{"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score":1356},{"slug":"kimi-k3","name":"Kimi K3","score":1336},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":1335},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":1324}]},{"key":"hle","name":"HLE","fullName":"Humanity's Last Exam","category":"knowledge","description":"An expert-authored benchmark designed to probe frontier knowledge and reasoning. BenchLM keeps protocol differences visible because tool-assisted and closed-book HLE runs answer different questions.","paperUrl":"https://lastexam.ai/","paperTitle":"Humanity's Last Exam","authors":"Center for AI Safety, Scale AI, and thousands of expert contributors","year":"2025","tasks":"Expert-level questions","format":"Open-ended and multiple choice","decimals":null,"modelCount":69,"lowerIsBetter":false,"top":[{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":65},{"slug":"claude-opus-5","name":"Claude Opus 5","score":64.7},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":64.5},{"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score":62.1},{"slug":"gpt-5-4-pro","name":"GPT-5.4 Pro","score":58.7}]},{"key":"swePro","name":"SWE-bench Pro","fullName":"SWE-bench Pro","category":"coding","description":"A long-horizon repository benchmark built to test realistic software engineering work. Its scores need a task-quality and setup check before they support a coding-agent decision.","paperUrl":"https://arxiv.org/abs/2509.16941","paperTitle":"SWE-Bench Pro: Can AI Agents Solve Long-Horizon Software Engineering Tasks?","authors":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","year":"2025","tasks":"1,865 repository problems","format":"Repository task completion","decimals":null,"modelCount":69,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":89.9},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":81.3},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":81.2},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":80.3},{"slug":"claude-fable","name":"Claude Fable 5","score":80}]},{"key":"terminalBench21","name":"Terminal-Bench 2.1","fullName":"Terminal-Bench 2.1","category":"coding","description":"Terminal-Bench 2.1 results, mostly provider-reported, stored separately from the Terminal-Bench 2.0 lane.","paperUrl":"https://www.tbench.ai/","paperTitle":"Terminal-Bench","authors":"Terminal-Bench contributors","year":"2026","tasks":"Terminal-based software-agent tasks","format":"Interactive task success rate","decimals":1,"modelCount":68,"lowerIsBetter":false,"top":[{"slug":"swe-2","name":"SWE-2","score":92.8},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":91.9},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":90.6},{"slug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","score":89.9},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":89.4}]},{"key":"terminalBench21","name":"Terminal-Bench 2.1","fullName":"Terminal-Bench 2.1","category":"agentic","description":"Terminal-Bench 2.1 results, mostly provider-reported, stored separately from the Terminal-Bench 2.0 lane.","paperUrl":"https://www.tbench.ai/","paperTitle":"Terminal-Bench","authors":"Terminal-Bench contributors","year":"2026","tasks":"Terminal-based software-agent tasks","format":"Interactive task success rate","decimals":1,"modelCount":68,"lowerIsBetter":false,"top":[{"slug":"swe-2","name":"SWE-2","score":92.8},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":91.9},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":90.6},{"slug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","score":89.9},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":89.4}]},{"key":"mmluPro","name":"MMLU-Pro","fullName":"Massive Multitask Language Understanding Professional","category":"knowledge","description":"An enhanced version of MMLU with 10 answer choices instead of 4, featuring more reasoning-focused questions that better differentiate frontier models.","paperUrl":"https://arxiv.org/abs/2406.01574","paperTitle":"MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark","authors":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","year":"2024","tasks":"Multiple subjects","format":"10-way multiple choice","decimals":null,"modelCount":67,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":89.6},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":89.5},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":88.5},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":88.5},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":87.8}]},{"key":"frontierMathV2Tiers13","name":"FrontierMath v2 (Tiers 1-3)","fullName":"FrontierMath v2 Tiers 1-3","category":"math","description":"Epoch AI's corrected v2 core FrontierMath suite of private advanced mathematics problems. Models can reason iteratively and use Python; scores are pass rates on the private set.","paperUrl":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","paperTitle":"FrontierMath v2 benchmark hub","authors":"Epoch AI","year":"2026","tasks":"285 private advanced mathematics problems","format":"Python-enabled iterative mathematical problem solving","decimals":3,"modelCount":64,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":93.684},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":93.684},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":91.228},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":90.175},{"slug":"gpt-6-sol","name":"GPT-6 Sol","score":89.825}]},{"key":"ifeval","name":"IFEval","fullName":"Instruction-Following Eval","category":"instructionFollowing","description":"A benchmark of 541 prompts built from 25 verifiable instruction types. It tests whether a model follows checkable constraints such as keyword, length, casing, and response-format requirements.","paperUrl":"https://arxiv.org/abs/2311.07911","paperTitle":"Instruction-Following Evaluation for Large Language Models","authors":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","year":"2023","tasks":"541 prompts across 25 instruction types","format":"Constrained generation","decimals":null,"modelCount":53,"lowerIsBetter":false,"top":[{"slug":"lfm2-5-24b-a2b","name":"LFM2.5-24B-A2B","score":96.75},{"slug":"qwen3-5-27b","name":"Qwen3.5-27B","score":95},{"slug":"agents-a1","name":"Agents-A1","score":94.82},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":94.6},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":94.3}]},{"key":"gertLabs","name":"Gert Labs","fullName":"Gert Labs Composite Game Benchmark","category":"agentic","description":"A game-environment benchmark that evaluates AI models in novel games covering strategic planning, resource management, spatial reasoning, cooperation, and theory of mind.","paperUrl":"https://gertlabs.com/rankings","paperTitle":"Gert Labs rankings","authors":"Gert Labs","year":"2026","tasks":"Novel game environments","format":"Composite game leaderboard","decimals":2,"modelCount":52,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":72.97},{"slug":"gpt-5-5","name":"GPT-5.5","score":72.93},{"slug":"claude-opus-4-7","name":"Claude Opus 4.7","score":65.59},{"slug":"gpt-5-4","name":"GPT-5.4","score":64.89},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":64.27}]},{"key":"frontierMathV2Tier4","name":"FrontierMath v2 (Tier 4)","fullName":"FrontierMath v2 Tier 4","category":"math","description":"Epoch AI's corrected v2 Tier 4 expansion, a separate set of exceptionally difficult research-level mathematics problems evaluated with Python-enabled iterative reasoning.","paperUrl":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2?view=graph&tab=leaderboard","paperTitle":"FrontierMath Tier 4 v2 leaderboard","authors":"Epoch AI","year":"2026","tasks":"41 private extreme-difficulty mathematics problems","format":"Python-enabled iterative mathematical problem solving","decimals":3,"modelCount":51,"lowerIsBetter":false,"top":[{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":100},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":97.6},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":95},{"slug":"claude-fable","name":"Claude Fable 5","score":90.2},{"slug":"gpt-6-sol","name":"GPT-6 Sol","score":90}]},{"key":"browseComp","name":"BrowseComp","fullName":"BrowseComp","category":"agentic","description":"BrowseComp tests whether a browsing agent can find difficult, fact-seeking answers across the web. Its 1,266 problems use short answers that are easier to grade than an open-ended research report.","paperUrl":"https://openai.com/index/browsecomp/","paperTitle":"BrowseComp","authors":"OpenAI","year":"2025","tasks":"Research questions requiring browsing","format":"Web search and evidence synthesis","decimals":null,"modelCount":50,"lowerIsBetter":false,"top":[{"slug":"atria-dawn-preview","name":"Atria Dawn Preview","score":92.5},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":92.2},{"slug":"ling-3-1-flash","name":"Ling 3.1 Flash","score":91.67},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":91.5},{"slug":"kimi-k3","name":"Kimi K3","score":91.2}]},{"key":"deepSwe","name":"DeepSWE","fullName":"DeepSWE","category":"coding","description":"A long-horizon software engineering benchmark from Datacurve for measuring frontier coding agents on original tasks drawn from active open-source repositories.","paperUrl":"https://deepswe.datacurve.ai/blog","paperTitle":"DeepSWE benchmark blog","authors":"Datacurve AI","year":"2026","tasks":"113 software engineering tasks across 91 repositories and 5 languages","format":"Pass@1 with confidence interval, cost, time, and token metadata","decimals":1,"modelCount":50,"lowerIsBetter":false,"top":[{"slug":"muse-wonder-full","name":"Muse Wonder Full","score":79},{"slug":"muse-wonder","name":"Muse Wonder","score":78.8},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":77.9},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":77},{"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score":75.4}]},{"key":"mmmuPro","name":"MMMU-Pro","fullName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodalGrounded","description":"A harder multimodal benchmark for frontier models that combines text with images, diagrams, charts, and academic visual reasoning tasks.","paperUrl":"https://arxiv.org/abs/2409.02813","paperTitle":"MMMU-Pro: A More Robust Multi-discipline Multimodal Understanding Benchmark","authors":"MMMU-Pro authors","year":"2024","tasks":"Multimodal academic reasoning","format":"Image + text question answering","decimals":null,"modelCount":48,"lowerIsBetter":false,"top":[{"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score":83.9},{"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score":83.6},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":83},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":82.3},{"slug":"kimi-k3","name":"Kimi K3","score":81.6}]},{"key":"liveCodeBenchV6","name":"LiveCodeBench v6","fullName":"LiveCodeBench v6","category":"coding","description":"LiveCodeBench v6 is a named release slice used in provider comparison tables. Keeping it separate prevents v6 results from being mixed into older or rolling LiveCodeBench windows.","paperUrl":"https://github.com/LiveCodeBench/LiveCodeBench","paperTitle":"LiveCodeBench official repository and release documentation","authors":"LiveCodeBench maintainers","year":"2026","tasks":"Fresh programming problems","format":"Provider-published v6 competitive programming results","decimals":1,"modelCount":46,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":92.6},{"slug":"solar-open-2","name":"Solar Open 2","score":92.4},{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":91.9},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":91.6},{"slug":"dots3-note-preview","name":"dots3-note Preview","score":91.5}]},{"key":"sweMultilingual","name":"SWE Multilingual","fullName":"SWE Multilingual","category":"coding","description":"A multilingual software-engineering benchmark for real-world code issue resolution across multiple programming languages.","paperUrl":"https://www.minimax.io/news/minimax-m27-en","paperTitle":"MiniMax M2.7: Early Echoes of Self-Evolution","authors":"MiniMax","year":"2026","tasks":"Multilingual software-engineering tasks","format":"Repository task completion","decimals":null,"modelCount":46,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":93.9},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":92.2},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":90.3},{"slug":"claude-opus-5","name":"Claude Opus 5","score":89.5},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":89.1}]},{"key":"sweMultilingual","name":"SWE Multilingual","fullName":"SWE-bench Multilingual","category":"multilingual","description":"A multilingual extension of SWE-bench covering 300 problems across 9 programming languages, testing code generation and bug fixing beyond Python.","paperUrl":"https://www.swebench.com/multilingual","paperTitle":"SWE-bench Multilingual","authors":"SWE-bench team","year":"2025","tasks":"300 problems across 9 languages","format":"Multi-language code patch generation","decimals":null,"modelCount":46,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":93.9},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":92.2},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":90.3},{"slug":"claude-opus-5","name":"Claude Opus 5","score":89.5},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":89.1}]},{"key":"hleNoTools","name":"HLE w/o tools","fullName":"Humanity's Last Exam without tools","category":"knowledge","description":"Tool-free variant of Humanity's Last Exam that isolates a model's raw frontier reasoning.","paperUrl":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","paperTitle":"Introducing GPT-5.4 mini and nano","authors":"OpenAI","year":"2026","tasks":"Expert-level questions","format":"Tool-free expert QA","decimals":null,"modelCount":43,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":64.4},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":60.9},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":59},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":56.9},{"slug":"claude-opus-5","name":"Claude Opus 5","score":56.3}]},{"key":"charxiv","name":"CharXiv","fullName":"CharXiv Reasoning","category":"multimodalGrounded","description":"A scientific chart reasoning benchmark that tests whether models can understand, interpret, and reason about complex scientific visualizations including plots, diagrams, and data charts.","paperUrl":"https://charxiv.github.io/","paperTitle":"CharXiv: Charting Gaps in Realistic Chart Understanding in Multimodal LLMs","authors":"CharXiv authors","year":"2024","tasks":"Scientific chart reasoning","format":"Chart understanding and reasoning","decimals":null,"modelCount":42,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":93.5},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":93.5},{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":91.4},{"slug":"kimi-k3","name":"Kimi K3","score":91.3},{"slug":"claude-opus-4-7-adaptive","name":"Claude Opus 4.7 (Adaptive)","score":91}]},{"key":"ifBench","name":"IFBench","fullName":"Instruction Following Benchmark","category":"instructionFollowing","description":"IFBench evaluates precise instruction-following generalization on 58 challenging, verifiable out-of-domain constraints. Unlike IFEval which tests familiar constraint types, IFBench specifically measures how well models follow novel instructions they haven't been optimized for, exposing overfitting to common instruction patterns.","paperUrl":null,"paperTitle":null,"authors":null,"year":2025,"tasks":58,"format":null,"decimals":null,"modelCount":42,"lowerIsBetter":false,"top":[{"slug":"mai-thinking-1","name":"MAI-Thinking-1","score":85},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":82.8},{"slug":"inkling-small","name":"Inkling-Small","score":82.2},{"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score":81.7},{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":81.5}]},{"key":"mcpAtlas","name":"MCP Atlas","fullName":"MCP Atlas","category":"agentic","description":"A benchmark for tool-calling over Model Context Protocol integrations and external tools.","paperUrl":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","paperTitle":"Introducing GPT-5.4 mini and nano","authors":"OpenAI","year":"2026","tasks":"Tool-integrated agent tasks","format":"Interactive tool-calling evaluation","decimals":null,"modelCount":42,"lowerIsBetter":false,"top":[{"slug":"muse-spark-open","name":"Muse Spark Open","score":89.2},{"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score":88.1},{"slug":"claude-opus-5","name":"Claude Opus 5","score":85.8},{"slug":"step-5-preview","name":"Step 5 Preview","score":85.6},{"slug":"kimi-k3","name":"Kimi K3","score":84.2}]},{"key":"mazeBench","name":"MazeBench","fullName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","description":"Tests whether a coding agent can explore a 256-room 3D puzzle world and collect its 100 hidden gems, using box-pushing puzzles, camera rotation, and long-horizon planning.","paperUrl":"https://mazebench.com/blog?post=maze-bench-results","paperTitle":"MazeBench Results","authors":"Jonathan Pappas and David Pappas","year":"2026","tasks":"100 hidden gems across 256 rooms in one open world","format":"Gems collected out of 100, with rooms visited as a second score","decimals":0,"modelCount":42,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":23},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":17},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":13},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":13},{"slug":"claude-opus-5","name":"Claude Opus 5","score":12}]},{"key":"arcAgi2","name":"ARC-AGI-2","fullName":"Abstraction and Reasoning Corpus for AGI v2","category":"reasoning","description":"ARC-AGI-2 tests whether a system can infer a transformation from example grids and apply it to a new grid. Compare accuracy with the evaluation split, attempt limit, and cost attached.","paperUrl":"https://arcprize.org/arc-agi/2/","paperTitle":"ARC-AGI-2: A Harder General Intelligence Benchmark","authors":"Francois Chollet, ARC Prize Foundation","year":2025,"tasks":"Visual pattern completion and abstract reasoning","format":"Grid transformation puzzles with novel rules","decimals":null,"modelCount":41,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":95},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":94.166667},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":92.5},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":91.666667},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":91.25}]},{"key":"terminalBench2","name":"Terminal-Bench 2.0","fullName":"Terminal-Bench 2.0","category":"coding","description":"A benchmark for agentic software engineering tasks executed in real terminal environments. DeepSeek reports it in the agentic section, while BenchLM also mirrors it in coding for models that publish it as a developer-task signal.","paperUrl":"https://www.tbench.ai/","paperTitle":"Terminal-Bench 2.0","authors":"Terminal-Bench contributors","year":"2026","tasks":"Terminal-based software tasks","format":"Interactive CLI agent evaluation","decimals":1,"modelCount":39,"lowerIsBetter":false,"top":[{"slug":"gpt-5-5","name":"GPT-5.5","score":82},{"slug":"gpt-5-3-codex","name":"GPT-5.3 Codex","score":77.3},{"slug":"gpt-5-4","name":"GPT-5.4","score":75.1},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":70.3},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":69.7}]},{"key":"terminalBench2","name":"Terminal-Bench 2.0","fullName":"Terminal-Bench 2.0","category":"agentic","description":"A benchmark for agentic software engineering tasks executed in real terminal environments. Models must inspect files, run commands, edit code, and recover from errors over multi-step workflows.","paperUrl":"https://www.tbench.ai/","paperTitle":"Terminal-Bench 2.0","authors":"Terminal-Bench contributors","year":"2026","tasks":"Terminal-based software tasks","format":"Interactive CLI agent evaluation","decimals":null,"modelCount":39,"lowerIsBetter":false,"top":[{"slug":"gpt-5-5","name":"GPT-5.5","score":82},{"slug":"gpt-5-3-codex","name":"GPT-5.3 Codex","score":77.3},{"slug":"gpt-5-4","name":"GPT-5.4","score":75.1},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":70.3},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":69.7}]},{"key":"aime2025","name":"AIME 2025","fullName":"American Invitational Mathematics Examination 2025","category":"math","description":"The most recent AIME examination, featuring 15 challenging mathematics problems testing olympiad-level mathematical reasoning with integer answers from 000-999.","paperUrl":"https://www.maa.org/math-competitions/aime","paperTitle":"American Invitational Mathematics Examination","authors":"Mathematical Association of America","year":"2025","tasks":"15 problems","format":"Integer answers 000-999","decimals":null,"modelCount":38,"lowerIsBetter":false,"top":[{"slug":"gpt-5-2","name":"GPT-5.2","score":100},{"slug":"step-3-5-flash","name":"Step 3.5 Flash","score":97.3},{"slug":"mai-thinking-1","name":"MAI-Thinking-1","score":97},{"slug":"kimi-k2-5-reasoning","name":"Kimi K2.5 (Reasoning)","score":96.1},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":96.1}]},{"key":"parseBenchLlamaIndex","name":"ParseBench","fullName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodalGrounded","description":"Scores document parsers and vision-language models on 2,078 human-verified enterprise pages across tables, charts, content faithfulness, semantic formatting and visual grounding, with the listed cost per page.","paperUrl":"https://arxiv.org/abs/2604.08538","paperTitle":"ParseBench: A Document Parsing Benchmark for AI Agents","authors":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","year":"2026","tasks":"2,078 human-verified enterprise pages from insurance, finance and government documents","format":"Deterministic test rules per page; five dimension scores and their mean, in percent","decimals":2,"modelCount":37,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":79.85},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":78.92},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":75.35},{"slug":"gemini-3-flash","name":"Gemini 3 Flash","score":75.05},{"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score":74.02}]},{"key":"arcAgi1","name":"ARC-AGI-1","fullName":"ARC-AGI-1 Semi-Private Evaluation","category":"reasoning","description":"ARC Prize fluid-intelligence benchmark using novel visual grid transformations.","paperUrl":"https://arcprize.org/","paperTitle":"ARC Prize leaderboard","authors":"ARC Prize Foundation","year":"2026","tasks":"Semi-private ARC-AGI-1 evaluation set","format":"Verified accuracy","decimals":2,"modelCount":36,"lowerIsBetter":false,"top":[{"slug":"claude-fable","name":"Claude Fable 5","score":98.5},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":98.5},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":98.5},{"slug":"claude-opus-5","name":"Claude Opus 5","score":97.5},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":97.5}]},{"key":"clawEval","name":"Claw-Eval","fullName":"Claw-Eval","category":"agentic","description":"A transparent real-world autonomous-agent benchmark with 300 human-verified tasks, 2,159 rubric items, and Pass^3 scoring across general, multi-turn, and native multimodal agent tasks.","paperUrl":"https://arxiv.org/abs/2604.06132","paperTitle":"Claw-Eval: Towards Trustworthy Evaluation of Autonomous Agents","authors":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","year":"2026","tasks":"300 tasks, 2,159 rubrics","format":"End-to-end autonomous-agent evaluation with Pass^3 scoring","decimals":1,"modelCount":36,"lowerIsBetter":false,"top":[{"slug":"ornith-1-5-397b","name":"Ornith-1.5-397B","score":81.4},{"slug":"k-exaone-2-0","name":"K-EXAONE 2.0","score":77.7},{"slug":"ornith-1-0-397b","name":"Ornith-1.0-397B","score":77.1},{"slug":"minimax-m3","name":"MiniMax M3","score":74.5},{"slug":"dots3-note-preview","name":"dots3-note Preview","score":73.4}]},{"key":"osWorldVerified","name":"OSWorld-Verified","fullName":"OSWorld-Verified","category":"agentic","description":"OSWorld-Verified is the July 2025 repaired release of OSWorld's real-computer evaluation. It measures whether a model-agent system can finish desktop and web tasks from configured starting states, with success checked by execution-based evaluators.","paperUrl":"https://os-world.github.io/","paperTitle":"OSWorld","authors":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","year":"2025","tasks":"369 real-world computer tasks (361 when eight Google Drive tasks are excluded)","format":"Execution-based interactive task success","decimals":null,"modelCount":35,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":86.1},{"slug":"claude-fable","name":"Claude Fable 5","score":85},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":85},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":84.3},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":83.4}]},{"key":"jobBench","name":"JobBench","fullName":"JobBench","category":"agentic","description":"An occupational agent benchmark for professional workflows that workers say they most want delegated to AI.","paperUrl":"https://arxiv.org/abs/2605.26329","paperTitle":"JobBench: Aligning Agent Work With Human Will","authors":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","year":"2026","tasks":"130 tasks across 35 occupations","format":"Agentic workplace deliverables","decimals":1,"modelCount":34,"lowerIsBetter":false,"top":[{"slug":"muse-wonder-full","name":"Muse Wonder Full","score":75.3},{"slug":"muse-wonder","name":"Muse Wonder","score":70.5},{"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score":64.9},{"slug":"ultramarine-1-flash","name":"Ultramarine 1 Flash","score":62.9},{"slug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","score":62}]},{"key":"nl2Repo","name":"NL2Repo","fullName":"NL2Repo","category":"coding","description":"A repository-understanding benchmark that measures whether models can map natural-language requests onto the right code locations and system changes.","paperUrl":"https://www.minimax.io/news/minimax-m27-en","paperTitle":"MiniMax M2.7: Early Echoes of Self-Evolution","authors":"MiniMax","year":"2026","tasks":"Natural language to repository tasks","format":"Repository understanding benchmark","decimals":null,"modelCount":31,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":68.2},{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":61.5},{"slug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","score":61.2},{"slug":"ornith-1-5-397b","name":"Ornith-1.5-397B","score":59.5},{"slug":"hy4-preview","name":"Hy4 preview","score":58.9}]},{"key":"aime2026","name":"AIME26","fullName":"AIME 2026","category":"math","description":"A 2026 American Invitational Mathematics Examination snapshot used in frontier-model comparison tables for mathematical reasoning.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Competition math problems","format":"Short-answer mathematics","decimals":1,"modelCount":29,"lowerIsBetter":false,"top":[{"slug":"glm-5-2","name":"GLM-5.2","score":99.2},{"slug":"reflection-beam","name":"Beam 1.0 501B-A23B","score":97.7},{"slug":"inkling","name":"Inkling","score":97.1},{"slug":"a-x-k2","name":"A.X K2","score":97.1},{"slug":"kimi-2-6","name":"Kimi K2.6","score":96.4}]},{"key":"mmlu","name":"MMLU","fullName":"Massive Multitask Language Understanding","category":"knowledge","description":"A comprehensive multiple-choice question answering test covering 57 tasks including elementary mathematics, US history, computer science, law, and more. Tests knowledge across diverse academic subjects from high school to professional level.","paperUrl":"https://arxiv.org/abs/2009.03300","paperTitle":"Measuring Massive Multitask Language Understanding","authors":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","year":"2020","tasks":"57 subjects","format":"Multiple choice questions","decimals":null,"modelCount":28,"lowerIsBetter":false,"top":[{"slug":"o1","name":"o1","score":91.8},{"slug":"deepseek-r1","name":"DeepSeek-R1","score":90.8},{"slug":"gpt-4-5","name":"GPT-4.5","score":90.8},{"slug":"sarvam-105b","name":"Sarvam 105B","score":90.6},{"slug":"gpt-4-1","name":"GPT-4.1","score":90.2}]},{"key":"hleWithTools","name":"HLE w/ tools","fullName":"Humanity's Last Exam with tools","category":"agentic","description":"Tool-augmented Humanity's Last Exam scores reported in DeepSeek-V4 thinking-mode evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Expert questions with tool use","format":"Pass@1","decimals":1,"modelCount":27,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":67.7},{"slug":"claude-opus-5","name":"Claude Opus 5","score":64.7},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":64.5},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":63.9},{"slug":"glm-5-3","name":"GLM-5.3","score":62.5}]},{"key":"automationBench","name":"AutomationBench","fullName":"AutomationBench","category":"agentic","description":"An agent benchmark for completing automation workflows in reproducible task environments.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI","year":"2026","tasks":"600 public automation tasks","format":"Agent task-completion score","decimals":1,"modelCount":25,"lowerIsBetter":false,"top":[{"slug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","score":62.2},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":60.5},{"slug":"atria-dawn-preview","name":"Atria Dawn Preview","score":53.8},{"slug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","score":53.1},{"slug":"ling-3-1-flash","name":"Ling 3.1 Flash","score":52.5}]},{"key":"cyberGym","name":"CyberGym","fullName":"CyberGym","category":"agentic","description":"A cybersecurity task benchmark for evaluating defensive cyber workflows and vulnerability-oriented agent performance.","paperUrl":"https://www.cybergym.io/","paperTitle":"CyberGym: Evaluating AI Agents' Real-World Cybersecurity Capabilities at Scale","authors":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","year":"2026","tasks":"1,507 vulnerability analysis instances","format":"Vulnerability reproduction and PoC generation","decimals":1,"modelCount":25,"lowerIsBetter":false,"top":[{"slug":"mimo-v2-6-flash","name":"MiMo-V2.6-Flash","score":95.1},{"slug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","score":94},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":88.1},{"slug":"ling-3-1-flash","name":"Ling 3.1 Flash","score":87.9},{"slug":"atria-dawn-preview","name":"Atria Dawn Preview","score":86.5}]},{"key":"extractBenchLlamaIndex","name":"ExtractBench (LlamaIndex)","fullName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodalGrounded","description":"Scores schema-guided extraction from 370 enterprise documents and 4,869 pages on value F1, record completeness, word and page grounding and cost, split by short, medium and long documents.","paperUrl":"https://arxiv.org/abs/2607.29677","paperTitle":"ExtractBench: A Benchmark for Schema-Guided Enterprise Document Extraction","authors":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","year":"2026","tasks":"370 documents, 4,869 pages, 8 business domains, 67 document types, each with a JSON Schema","format":"Order-insensitive value F1 overall and by document length; word and page grounding F1; cost and latency","decimals":2,"modelCount":24,"lowerIsBetter":true,"top":[{"slug":"gemma-4-12b","name":"Gemma 4 12B","score":45.39},{"slug":"gemma-4-e2b","name":"Gemma 4 E2B","score":51.85},{"slug":"gemma-4-e4b","name":"Gemma 4 E4B","score":69.64},{"slug":"gemma-4-26b-a4b","name":"Gemma 4 26B A4B","score":70.27},{"slug":"gpt-5-4-nano","name":"GPT-5.4 nano","score":74.9}]},{"key":"hmmtFeb2026","name":"HMMT Feb 2026","fullName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","description":"A February 2026 HMMT slice used in newer frontier-model math comparisons.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Competition math problems","format":"Contest mathematics","decimals":1,"modelCount":24,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":97.1},{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":95.2},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":94.8},{"slug":"solar-open-2","name":"Solar Open 2","score":93.9},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":92.9}]},{"key":"osWorld2","name":"OSWorld 2.0","fullName":"OSWorld 2.0","category":"agentic","description":"OSWorld 2.0 evaluates long-horizon computer-use tasks. Published results use different task releases, metrics, and run settings, so scores appear by comparable protocol and unmatched rows stay unranked.","paperUrl":"https://arxiv.org/abs/2606.29537","paperTitle":"OSWorld2.0: Benchmarking Computer Use Agents on Long-Horizon Real-World Tasks","authors":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","year":"2026","tasks":"108 long-horizon computer-use workflows","format":"Interactive computer-use evaluation; binary completion and partial-score metrics","decimals":1,"modelCount":24,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":72.6},{"slug":"claude-opus-5","name":"Claude Opus 5","score":70.6},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":69.2},{"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score":66.9},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":62.6}]},{"key":"bug-hunt-bench","name":"Bug Hunt Bench","fullName":"Bug Hunt Bench","category":"coding","description":"A blind-graded coding-agent benchmark with 105 planted bugs across two production TypeScript repositories.","paperUrl":"https://bughunt.productcompass.pm/method","paperTitle":"Bug Hunt Bench method, caveats, and definitions","authors":"Pawel Huryn","year":"2026","tasks":"105 planted bugs across two production TypeScript repositories","format":"Strict planted bugs fixed","decimals":1,"modelCount":23,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":51.3},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":45},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":44.3},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":43},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":42}]},{"key":"sciCode","name":"SciCode","fullName":"Scientific Code Benchmark","category":"coding","description":"SciCode evaluates language models on generating code for realistic scientific research problems across 16 subfields of physics, math, chemistry, biology, and material science. Problems decompose into 338 subproblems requiring domain knowledge recall, scientific reasoning, and precise code synthesis. Based on real scripts from published research.","paperUrl":null,"paperTitle":null,"authors":null,"year":2024,"tasks":80,"format":null,"decimals":null,"modelCount":23,"lowerIsBetter":false,"top":[{"slug":"step-5-preview","name":"Step 5 Preview","score":58.9},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":53.5},{"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score":53.1},{"slug":"kimi-2-6","name":"Kimi K2.6","score":52.2},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":51.3}]},{"key":"tau3Bench","name":"τ³-bench results","fullName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","description":"τ³-bench is the current evolution of Sierra's tool-agent-user framework, adding corrected task releases and newer knowledge and voice evaluation modes alongside airline, retail, and telecom.","paperUrl":"https://github.com/sierra-research/tau2-bench","paperTitle":"Official τ³-bench repository and release notes","authors":"Sierra Research","year":"2026","tasks":"Corrected customer-service tasks plus knowledge and voice evaluation modes","format":"Published domain or average success results","decimals":1,"modelCount":23,"lowerIsBetter":false,"top":[{"slug":"mercury-2-5","name":"Mercury 2.5","score":96},{"slug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5 128B","score":91.4},{"slug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro","score":72.9},{"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score":70.9},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":70.7}]},{"key":"arcAgi3","name":"ARC-AGI-3","fullName":"Abstraction and Reasoning Corpus for AGI v3","category":"reasoning","description":"An interactive successor to ARC-AGI-2 that evaluates whether an AI agent can learn unfamiliar task mechanics through action and feedback.","paperUrl":"https://arcprize.org/media/ARC_AGI_3_Technical_Report.pdf","paperTitle":"ARC-AGI-3: A New Challenge for Frontier Agentic Intelligence","authors":"ARC Prize Foundation","year":2026,"tasks":"Interactive game-like tasks with hidden rules","format":"Agentic task completion under a capped evaluation budget","decimals":null,"modelCount":22,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":62.712802},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":52.72904},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":46.265668},{"slug":"claude-opus-5","name":"Claude Opus 5","score":30.16},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":10.366279}]},{"key":"toolathlon","name":"Toolathlon","fullName":"Toolathlon","category":"agentic","description":"A tool-use benchmark focused on selecting, sequencing, and completing tasks with external tools.","paperUrl":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","paperTitle":"Introducing GPT-5.4 mini and nano","authors":"OpenAI","year":"2026","tasks":"Multi-tool workflows","format":"Interactive tool-calling evaluation","decimals":null,"modelCount":22,"lowerIsBetter":false,"top":[{"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score":75.6},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":59.9},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":58},{"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score":56.5},{"slug":"gpt-5-5","name":"GPT-5.5","score":55.6}]},{"key":"toolathlonVerified","name":"Toolathlon-Verified","fullName":"Toolathlon-Verified","category":"agentic","description":"A verified tool-use benchmark variant for completing multi-step workflows with external tools.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI","year":"2026","tasks":"Verified multi-tool workflows","format":"Interactive tool-use score","decimals":1,"modelCount":22,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":80.6},{"slug":"glm-5-3-flash","name":"GLM-5.3-Flash","score":78.4},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":77.8},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":77.8},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":77.8}]},{"key":"mmmu","name":"MMMU","fullName":"Massive Multi-discipline Multimodal Understanding","category":"multimodalGrounded","description":"A broad multimodal reasoning benchmark spanning charts, diagrams, tables, and academic visual question answering.","paperUrl":"https://arxiv.org/abs/2401.05508","paperTitle":"MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI","authors":"MMMU authors","year":"2024","tasks":"Multimodal academic reasoning","format":"Image + text question answering","decimals":1,"modelCount":21,"lowerIsBetter":false,"top":[{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":87.2},{"slug":"gpt-5-2","name":"GPT-5.2","score":86.7},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":86},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":85},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":84.3}]},{"key":"deepSearchQa","name":"DeepSearchQA","fullName":"DeepSearchQA","category":"agentic","description":"An agentic browsing benchmark where models search the web, gather evidence, and answer list-style questions using browser tools.","paperUrl":"https://ai.meta.com/static-resource/muse-spark-eval-methodology","paperTitle":"Muse Spark Eval Methodology","authors":"Meta AI","year":"2026","tasks":"Agentic browsing and list-answer questions","format":"Search / open / find browser-agent evaluation","decimals":1,"modelCount":20,"lowerIsBetter":false,"top":[{"slug":"atria-dawn-preview","name":"Atria Dawn Preview","score":96},{"slug":"claude-opus-5","name":"Claude Opus 5","score":95},{"slug":"kimi-k3","name":"Kimi K3","score":95},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":93.1},{"slug":"step-3-7-flash","name":"Step 3.7 Flash","score":92.82}]},{"key":"frontierSweV2","name":"FrontierSWE v2","fullName":"FrontierSWE v2","category":"coding","description":"A 34-task expansion of FrontierSWE for ultra-long-horizon engineering and research work that remains far from saturation.","paperUrl":"https://www.frontierswe.com/blog/v2","paperTitle":"FrontierSWE v2","authors":"Proximal","year":"2026","tasks":"34 ultra-long-horizon engineering and research tasks","format":"Five-trial mean task score (Mean@5), 0-100","decimals":1,"modelCount":20,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":65.5},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":62.3},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":61.9},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":56.3},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":55}]},{"key":"math500","name":"MATH-500","fullName":"MATH-500 Problem Set","category":"math","description":"A curated subset of 500 problems from the MATH dataset, covering algebra, counting and probability, geometry, intermediate algebra, number theory, prealgebra, and precalculus.","paperUrl":"https://arxiv.org/abs/2103.03874","paperTitle":"Measuring Mathematical Problem Solving With the MATH Dataset","authors":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","year":"2021","tasks":"500 problems","format":"Free-form mathematical answers","decimals":null,"modelCount":20,"lowerIsBetter":false,"top":[{"slug":"ternary-bonsai-2-27b","name":"Ternary Bonsai 2 27B","score":98.8},{"slug":"sarvam-105b","name":"Sarvam 105B","score":98.6},{"slug":"qwq-32b","name":"QwQ-32B","score":98},{"slug":"kimi-k2","name":"Kimi K2","score":97.4},{"slug":"deepseek-r1","name":"DeepSeek-R1","score":97.3}]},{"key":"mathVision","name":"MathVision","fullName":"MathVision","category":"multimodalGrounded","description":"A visual mathematics benchmark that tests whether a model can solve math problems grounded in diagrams, equations, figures, and other visual inputs.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Visually grounded math problems","format":"Image + math reasoning","decimals":1,"modelCount":20,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":95.2},{"slug":"kimi-k3","name":"Kimi K3","score":94.3},{"slug":"seed-2-1-pro","name":"Seed 2.1 Pro","score":92.6},{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":91.8},{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":90.6}]},{"key":"superGpqa","name":"SuperGPQA","fullName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","description":"An expanded version of GPQA that evaluates graduate-level knowledge and reasoning capabilities across 285 disciplines, providing comprehensive coverage of academic domains.","paperUrl":"https://arxiv.org/abs/2502.14739","paperTitle":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","authors":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","year":"2025","tasks":"285 disciplines","format":"Multiple choice questions","decimals":null,"modelCount":20,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score":95},{"slug":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","score":95},{"slug":"qwen3-6-max-preview","name":"Qwen 3.6 Max (preview)","score":73.9},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":73.6},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":71.6}]},{"key":"aime2024","name":"AIME 2024","fullName":"American Invitational Mathematics Examination 2024","category":"math","description":"The 2024 edition of AIME, maintaining the same format of 15 challenging mathematics problems with integer answers from 000 to 999.","paperUrl":"https://www.maa.org/math-competitions/aime","paperTitle":"American Invitational Mathematics Examination","authors":"Mathematical Association of America","year":"2024","tasks":"15 problems","format":"Integer answers 000-999","decimals":null,"modelCount":19,"lowerIsBetter":false,"top":[{"slug":"gpt-oss-120b","name":"GPT-OSS 120B","score":95.8},{"slug":"deepseek-v3-1-reasoning","name":"DeepSeek V3.1 (Reasoning)","score":93.1},{"slug":"gpt-oss-20b","name":"GPT-OSS 20B","score":92.1},{"slug":"gemini-2-5-pro","name":"Gemini 2.5 Pro","score":92},{"slug":"grok-3-mini","name":"Grok 3 Mini","score":89.5}]},{"key":"bfclV4","name":"BFCL v4","fullName":"Berkeley Function Calling Leaderboard v4","category":"agentic","description":"A function-calling benchmark for tool selection, schema adherence, and argument correctness.","paperUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","paperTitle":"The Berkeley Function Calling Leaderboard (BFCL): From Tool Use to Agentic Evaluation of Large Language Models","authors":"Shishir G. Patil et al.","year":"2025","tasks":"Function-calling tasks","format":"Tool invocation and schema evaluation","decimals":1,"modelCount":19,"lowerIsBetter":false,"top":[{"slug":"atria-dawn-preview","name":"Atria Dawn Preview","score":77},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":75},{"slug":"btl-4","name":"BTL-4","score":73.5},{"slug":"ling-3-0-flash","name":"Ling 3.0 Flash","score":73},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":72.9}]},{"key":"researchClawBench","name":"ResearchClawBench","fullName":"ResearchClawBench","category":"agentic","description":"An end-to-end autonomous scientific research benchmark with 40 tasks across 10 scientific domains, where agents receive related literature and raw data, then attempt to rediscover the hidden target paper.","paperUrl":"https://arxiv.org/abs/2606.07591","paperTitle":"ResearchClawBench: A Benchmark for End-to-End Autonomous Scientific Research","authors":"InternScience","year":"2026","tasks":"40 tasks across 10 scientific domains","format":"End-to-end autonomous research evaluation with RADS scoring","decimals":1,"modelCount":19,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":21.1},{"slug":"glm-5-2","name":"GLM-5.2","score":20.7},{"slug":"claude-opus-4-7","name":"Claude Opus 4.7","score":20.7},{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score":19.9},{"slug":"minimax-m3","name":"MiniMax M3","score":19.8}]},{"key":"apprenticeBench","name":"ApprenticeBench","fullName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","description":"Tests whether a computer-use agent can learn a real accounts-payable job on the job, processing 100 vendor bills in a company ERP system with only the handbook, historical records, and mentor feedback a new hire would get.","paperUrl":"https://neocognition.io/blog/apprentice-bench/","paperTitle":"ApprenticeBench: a step change in AI's job readiness","authors":"NeoCognition","year":"2026","tasks":"100 vendor bills processed in sequence inside a simulated construction company","format":"Cumulative success rate over 100 bills","decimals":0,"modelCount":18,"lowerIsBetter":false,"top":[{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":72},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":68},{"slug":"claude-opus-5","name":"Claude Opus 5","score":36},{"slug":"claude-fable","name":"Claude Fable 5","score":34},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":26}]},{"key":"humaneval","name":"HumanEval","fullName":"Evaluating Large Language Models Trained on Code","category":"coding","description":"HumanEval tests whether generated Python functions pass unit tests on 164 handwritten problems. These scores describe function-level correctness; they do not measure repository work or a coding agent.","paperUrl":"https://arxiv.org/abs/2107.03374","paperTitle":"Evaluating Large Language Models Trained on Code","authors":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","year":"2021","tasks":"164 problems","format":"Python function generation","decimals":null,"modelCount":18,"lowerIsBetter":false,"top":[{"slug":"o1-mini","name":"o1-mini","score":92.4},{"slug":"sarvam-30b","name":"Sarvam 30B","score":92.1},{"slug":"qwen2-5-vl-32b","name":"Qwen2.5-VL-32B","score":91.5},{"slug":"deepseek-v2-5","name":"DeepSeek-V2.5","score":89},{"slug":"gpt-4-5","name":"GPT-4.5","score":88.6}]},{"key":"terminalBench4","name":"Terminal-Bench 4.0","fullName":"Terminal-Bench 4.0","category":"agentic","description":"The current Terminal-Bench release measures difficult computer work after recalibrating task resources, fixing unstable tasks, and removing tasks that no longer separate frontier systems.","paperUrl":"https://www.tbench.ai/news/terminal-bench-4-0","paperTitle":"Terminal-Bench 4.0","authors":"Ryan Marten, Terminal-Bench contributors, and Harbor","year":"2026","tasks":"66 professional computer-work tasks","format":"Task completion rate across 5 trials per task","decimals":2,"modelCount":18,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":70.6},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":66.4},{"slug":"claude-mythos-5-1","name":"Claude Mythos 5.1","score":60.9},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":57.9},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":57.4}]},{"key":"agentsLastExam","name":"Agents' Last Exam","fullName":"Agents' Last Exam","category":"agentic","description":"An agent benchmark reported in DeepSeek's V4 Flash 0731 launch comparison.","paperUrl":"https://api-docs.deepseek.com/zh-cn/updates/","paperTitle":"DeepSeek V4 Flash 0731 update","authors":"DeepSeek-AI","year":"2026","tasks":"Agent tasks","format":"Provider-reported task score","decimals":1,"modelCount":17,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":59.3},{"slug":"gpt-6-sol","name":"GPT-6 Sol","score":56.4},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":52.4},{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":51.2},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":42.9}]},{"key":"longBenchV2","name":"LongBench v2","fullName":"LongBench v2","category":"reasoning","description":"A long-context benchmark that measures whether models can actually use extended context windows for reasoning and retrieval.","paperUrl":"https://arxiv.org/abs/2412.15204","paperTitle":"LongBench v2","authors":"LongBench v2 authors","year":"2025","tasks":"Long-context tasks","format":"Extended-context retrieval and reasoning","decimals":null,"modelCount":17,"lowerIsBetter":false,"top":[{"slug":"reflection-beam","name":"Beam 1.0 501B-A23B","score":67.5},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":66.3},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":64.4},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":63.2},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":62}]},{"key":"simpleVqa","name":"SimpleVQA","fullName":"SimpleVQA","category":"multimodalGrounded","description":"A visual question answering benchmark focused on straightforward image-grounded understanding.","paperUrl":"https://docs.z.ai/guides/vlm/glm-5v-turbo","paperTitle":"GLM-5V-Turbo","authors":"Z.AI","year":"2026","tasks":"Visual QA tasks","format":"Image-grounded question answering","decimals":1,"modelCount":17,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":81.7},{"slug":"step-3-7-flash","name":"Step 3.7 Flash","score":79.2},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":75},{"slug":"muse-spark-open","name":"Muse Spark Open","score":73.6},{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":73.2}]},{"key":"erqa","name":"ERQA","fullName":"ERQA","category":"multimodalGrounded","description":"A grounded visual reasoning benchmark focused on evidence-based question answering over real images.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Evidence-based visual QA","format":"Grounded image reasoning","decimals":1,"modelCount":16,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":77.8},{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":72.3},{"slug":"seed-2-1-pro","name":"Seed 2.1 Pro","score":72},{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":71},{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":70.5}]},{"key":"frontierCode","name":"FrontierCode 1.1 Main","fullName":"FrontierCode 1.1 Main","category":"coding","description":"Cognition's 100-task software-engineering benchmark for whether coding agents produce mergeable, production-quality pull requests, scored for correctness, tests, scope, style, and maintainability through maintainer-authored rubrics.","paperUrl":"https://cognition.com/frontiercode","paperTitle":"FrontierCode leaderboard","authors":"Cognition","year":"2026","tasks":"100 private Main tasks (150 in Extended)","format":"Repository task completion with maintainer rubrics","decimals":1,"modelCount":16,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":54.4},{"slug":"claude-fable","name":"Claude Fable 5","score":53.5},{"slug":"claude-opus-5","name":"Claude Opus 5","score":53.4},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":53.3},{"slug":"swe-2","name":"SWE-2","score":50}]},{"key":"healthBenchProfessional","name":"HealthBench Professional","fullName":"HealthBench Professional","category":"knowledge","description":"An open benchmark for clinician-facing model responses across care consult, writing and documentation, and medical research tasks.","paperUrl":"https://arxiv.org/abs/2604.27470","paperTitle":"HealthBench Professional: Evaluating Large Language Models on Real Clinician Chats","authors":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","year":"2026","tasks":"Clinician chat tasks","format":"Rubric-graded open-ended responses","decimals":1,"modelCount":16,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":69.2},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":65.6},{"slug":"ling-3-1-flash","name":"Ling 3.1 Flash","score":65.35},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":64.7},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":64.2}]},{"key":"hermesBench","name":"Hermes Bench","fullName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","description":"Tests agents on 150 tasks across 25 categories, including Hermes skills, research, diagrams and art, memory, tool use and safety. Each agent starts in a workspace with real files, some tasks add follow-up turns, and the grader checks the files and state left behind.","paperUrl":"https://portal.nousresearch.com/bench","paperTitle":"Hermes Bench (Hermes Index)","authors":"Nous Research","year":"2026","tasks":"150 tasks across 25 categories","format":"pass@1 score","decimals":1,"modelCount":16,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":76.73},{"slug":"grok-4-7","name":"Grok 4.7","score":74.38},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":73.93},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":73.67},{"slug":"gpt-6-sol","name":"GPT-6 Sol","score":73.5}]},{"key":"hermesIndex","name":"Hermes Index","fullName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","description":"An index of agentic capability from Nous Research. It averages pass@1 scores on four suites, Hermes Bench, TerminalBench 4, TerminalBench Science and SkillsBench, with every run inside the Hermes Agent harness, and reports the mean cost per task beside the score.","paperUrl":"https://portal.nousresearch.com/bench","paperTitle":"Hermes Index: an agentic model leaderboard","authors":"Nous Research","year":"2026","tasks":"Four suites: Hermes Bench (150 tasks), TerminalBench 4 without its 4 GPU tasks, TerminalBench Science and SkillsBench","format":"Mean pass@1 score across four suites, with mean cost per task","decimals":1,"modelCount":16,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":63.31},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":57.09},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":56.25},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":53.14},{"slug":"gpt-6-sol","name":"GPT-6 Sol","score":44.1}]},{"key":"mmluProX","name":"MMLU-ProX","fullName":"MMLU-ProX","category":"multilingual","description":"A multilingual extension of professional-level academic evaluation across many languages.","paperUrl":"https://arxiv.org/abs/2503.10497","paperTitle":"MMLU-ProX: A Multilingual Benchmark for Advanced Large Language Model Evaluation","authors":"MMLU-ProX authors","year":"2025","tasks":"Multilingual professional QA","format":"Multilingual multiple choice","decimals":null,"modelCount":16,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":87},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":85.7},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":85.4},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":84.7},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":84.7}]},{"key":"realWorldQa","name":"RealWorldQA","fullName":"RealWorldQA","category":"multimodalGrounded","description":"A grounded visual QA benchmark focused on answering practical questions about real-world images and scenes.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Real-world visual question answering","format":"Image-grounded QA","decimals":1,"modelCount":16,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":88.5},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":88},{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":87.7},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":86.9},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":85.9}]},{"key":"wideResearch","name":"WideResearch","fullName":"WideResearch","category":"agentic","description":"A broad research-agent benchmark for open-ended information gathering, synthesis, and answer construction across wide search spaces.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Open-ended research tasks","format":"Multi-source research evaluation","decimals":1,"modelCount":16,"lowerIsBetter":false,"top":[{"slug":"hy4-preview","name":"Hy4 preview","score":83.9},{"slug":"ling-3-1-flash","name":"Ling 3.1 Flash","score":83.32},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":81.9},{"slug":"atria-dawn-preview","name":"Atria Dawn Preview","score":81.9},{"slug":"kimi-2-6","name":"Kimi K2.6","score":80.8}]},{"key":"mmluRedux","name":"MMLU-Redux","fullName":"MMLU-Redux","category":"knowledge","description":"A harder refresh of MMLU intended to keep broad knowledge evaluation useful after the original benchmark became too easy for frontier models.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Broad academic QA","format":"Multiple choice questions","decimals":null,"modelCount":15,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":96.6},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":95},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":94.9},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":94.5},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":94.5}]},{"key":"postTrainBench11","name":"PostTrainBench v1.1","fullName":"PostTrainBench v1.1","category":"coding","description":"Post-training four base language models across seven weighted benchmarks, with ten hours and one H100 per run.","paperUrl":"https://posttrainbench.com/?version=v1.1","paperTitle":"PostTrainBench v1.1 leaderboard and methodology","authors":"PostTrainBench team","year":"2026","tasks":"Post-training Qwen3 1.7B and 4B, SmolLM3 3B, and Gemma3 4B","format":"Weighted aggregate across four base models and seven benchmarks","decimals":1,"modelCount":15,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":49.3},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":45.3},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":44.3},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":40.2},{"slug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","score":39.3}]},{"key":"programBench","name":"ProgramBench","fullName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","description":"A cleanroom software-engineering benchmark where agents receive only a compiled executable and documentation, then must architect and implement a complete codebase that reproduces the original program's behavior.","paperUrl":"https://programbench.com/static/paper.pdf","paperTitle":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","authors":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","year":"2026","tasks":"200 program reconstruction tasks","format":"Cleanroom executable reimplementation","decimals":1,"modelCount":15,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":93},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":91.2},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":87.6},{"slug":"step-5-preview","name":"Step 5 Preview","score":80.5},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":79.7}]},{"key":"reactNativeEvals","name":"React Native Evals","fullName":"React Native Evals","category":"coding","description":"An open benchmark for AI coding agents on real-world React Native implementation tasks, emphasizing working app behavior, recommended architecture choices, and strict constraint adherence.","paperUrl":"https://rn-evals.vercel.app/","paperTitle":"React Native Evals","authors":"Callstack","year":"2026","tasks":"React Native app implementation tasks","format":"Framework-specific app development evaluation","decimals":null,"modelCount":15,"lowerIsBetter":false,"top":[{"slug":"composer-2","name":"Composer 2","score":96.1},{"slug":"gpt-5-4","name":"GPT-5.4","score":85.3},{"slug":"gpt-5-5","name":"GPT-5.5","score":84.7},{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score":84.1},{"slug":"claude-opus-4-7","name":"Claude Opus 4.7","score":82.8}]},{"key":"screenSpotPro","name":"ScreenSpot Pro","fullName":"ScreenSpot Pro","category":"multimodalGrounded","description":"A GUI-grounding benchmark for 1,581 instructions in full-screen, high-resolution professional interfaces. It tests where a target is, not whether an agent can finish the surrounding workflow.","paperUrl":"https://arxiv.org/abs/2504.07981","paperTitle":"ScreenSpot-Pro: GUI Grounding for Professional High-Resolution Computer Use","authors":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","year":"2025","tasks":"1,581 grounding instructions","format":"Static interface element localization","decimals":1,"modelCount":15,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":92.7},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":87.9},{"slug":"gpt-5-4","name":"GPT-5.4","score":85.4},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":84.5},{"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score":84.4}]},{"key":"charxivNoTools","name":"CharXiv w/o tools","fullName":"CharXiv Reasoning without tools","category":"multimodalGrounded","description":"Tool-free variant of CharXiv that isolates raw visual reasoning ability without code execution or tool augmentation.","paperUrl":"https://charxiv.github.io/","paperTitle":"CharXiv: Charting Gaps in Realistic Chart Understanding in Multimodal LLMs","authors":"CharXiv authors","year":"2024","tasks":"Scientific chart reasoning (tool-free)","format":"Chart understanding without tools","decimals":null,"modelCount":14,"lowerIsBetter":false,"top":[{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":88.9},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":88.4},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":86.2},{"slug":"kimi-k3","name":"Kimi K3","score":84.8},{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":84.6}]},{"key":"exploitGym","name":"ExploitGym","fullName":"ExploitGym","category":"agentic","description":"A controlled benchmark for evaluating whether AI agents can extend vulnerability-triggering inputs into working exploits.","paperUrl":"https://arxiv.org/abs/2605.11086","paperTitle":"ExploitGym: Can AI Agents Turn Security Vulnerabilities into Real Attacks?","authors":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","year":"2026","tasks":"898 exploitation tasks","format":"Working exploit generation","decimals":1,"modelCount":14,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":42.4},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":35.1},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":33.7},{"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","score":23.2},{"slug":"gpt-6-sol","name":"GPT-6 Sol","score":22.1}]},{"key":"vulcanBench","name":"VulcanBench v3","fullName":"VulcanBench v3","category":"coding","description":"An open software-engineering benchmark built from real merged post-cutoff pull requests across Python, Rust, TypeScript, JavaScript, and Go repositories.","paperUrl":"https://github.com/morganlinton/VulcanBench/tree/main","paperTitle":"VulcanBench","authors":"VulcanBench contributors","year":"2026","tasks":"23 post-cutoff repository tasks in the v3 report","format":"Pass@1 with low, medium, and high effort","decimals":1,"modelCount":14,"lowerIsBetter":false,"top":[{"slug":"grok-4-5","name":"Grok 4.5","score":89.9},{"slug":"claude-fable","name":"Claude Fable 5","score":89.5},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":88.4},{"slug":"claude-opus-5","name":"Claude Opus 5","score":87},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":87}]},{"key":"hmmtFeb2025","name":"HMMT Feb 2025","fullName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","description":"A February 2025 HMMT slice used in exact-value provider tables for advanced contest-math reasoning.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2025","tasks":"Competition math problems","format":"Contest mathematics","decimals":1,"modelCount":13,"lowerIsBetter":false,"top":[{"slug":"glm-5","name":"GLM-5","score":97.5},{"slug":"glm-4-7","name":"GLM-4.7","score":97.1},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":96.7},{"slug":"kimi-k2-5-reasoning","name":"Kimi K2.5 (Reasoning)","score":95.4},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":95.4}]},{"key":"officeQaPro","name":"OfficeQA Pro","fullName":"OfficeQA Pro","category":"multimodalGrounded","description":"A benchmark for grounded reasoning over office-style documents, spreadsheets, charts, and business artifacts.","paperUrl":"https://arxiv.org/abs/2603.08655","paperTitle":"OfficeQA Pro: An Enterprise Benchmark for End-to-End Grounded Reasoning","authors":"OfficeQA Pro authors","year":"2026","tasks":"Document and spreadsheet tasks","format":"Grounded QA over office artifacts","decimals":null,"modelCount":13,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":67.7},{"slug":"claude-opus-5","name":"Claude Opus 5","score":66.9},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":66.2},{"slug":"hy4-preview","name":"Hy4 preview","score":66.2},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":65.6}]},{"key":"sweRebench","name":"SWE-Rebench","fullName":"SWE-Rebench","category":"coding","description":"A continuously updated software engineering benchmark by Nebius using fresh GitHub issues to avoid contamination. Models are evaluated 5 times per problem under a fixed ReAct scaffolding; the Resolved Rate (best pass@1) is reported.","paperUrl":"https://swe-rebench.com","paperTitle":"SWE-Rebench: Contamination-Free Evaluation of Software Engineering Agents","authors":"Nebius","year":"2026","tasks":"Fresh GitHub issues (rolling window)","format":"Code patch generation","decimals":null,"modelCount":13,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score":65.3},{"slug":"glm-5","name":"GLM-5","score":62.8},{"slug":"glm-5-1","name":"GLM-5.1","score":62.7},{"slug":"deepseek-v3-2","name":"DeepSeek V3.2","score":60.9},{"slug":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","score":60.7}]},{"key":"frontierBench","name":"Terminal-Bench 3.0","fullName":"Terminal-Bench 3.0","category":"agentic","description":"A continuously maintained benchmark for difficult computer work, including coding, deep learning, finance, engineering, math, and science tasks.","paperUrl":"https://www.frontierbench.ai/","paperTitle":"Terminal-Bench 3.0","authors":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","year":"2026","tasks":"74 professional computer-work tasks across 7 domains","format":"Task completion rate","decimals":1,"modelCount":13,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":42.7},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":34.59},{"slug":"claude-fable","name":"Claude Fable 5","score":34.05},{"slug":"grok-4-6","name":"Grok 4.6","score":26.49},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":21.08}]},{"key":"graySwanIpi15","name":"Gray Swan IPI (15 attempts)","fullName":"Gray Swan Indirect Prompt Injection, attack success at 15 attempts","category":"instructionFollowing","description":"Measures how often indirect prompt-injection attacks succeed within fifteen attempts. Lower scores mean greater resistance to the tested attacks.","paperUrl":"https://deepmind.google/models/evals-methodology/gemini-4-argon","paperTitle":"Gemini 4 Argon evaluation methodology","authors":"Gray Swan","year":"2026","tasks":"Indirect prompt-injection attacks","format":"Attack success rate at 15 attempts","decimals":1,"modelCount":12,"lowerIsBetter":true,"top":[{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":0.7},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":1},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":1},{"slug":"claude-opus-5","name":"Claude Opus 5","score":4.6},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":5.5}]},{"key":"hmmtNov2025","name":"HMMT Nov 2025","fullName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","description":"A November 2025 HMMT slice for high-end mathematical reasoning comparisons.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2025","tasks":"Competition math problems","format":"Contest mathematics","decimals":1,"modelCount":12,"lowerIsBetter":false,"top":[{"slug":"glm-5","name":"GLM-5","score":96.9},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":94.6},{"slug":"glm-5-2","name":"GLM-5.2","score":94.4},{"slug":"glm-5-1","name":"GLM-5.1","score":94},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":93.3}]},{"key":"healthBenchHard","name":"HealthBench Hard","fullName":"HealthBench Hard","category":"knowledge","description":"A harder subset of OpenAI's HealthBench for evaluating open-ended medical and health reasoning with rubric-based grading.","paperUrl":"https://ai.meta.com/static-resource/muse-spark-eval-methodology","paperTitle":"Muse Spark Eval Methodology","authors":"Meta AI","year":"2026","tasks":"1,000 health prompts","format":"Open-ended health evaluation","decimals":1,"modelCount":12,"lowerIsBetter":false,"top":[{"slug":"muse-spark","name":"Muse Spark","score":42.8},{"slug":"gpt-5-4","name":"GPT-5.4","score":40.1},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":36.6},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":36.2},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":33.1}]},{"key":"mrcrv2","name":"MRCRv2","fullName":"MRCRv2","category":"reasoning","description":"A long-context benchmark for memory, retrieval, and multi-round coherence over large contexts.","paperUrl":"https://openai.com/index/introducing-gpt-5-2/","paperTitle":"Introducing GPT-5.2 and GPT-5.2 Pro","authors":"OpenAI","year":"2025","tasks":"Long-context retrieval","format":"Multi-round long-context evaluation","decimals":null,"modelCount":12,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":92.9},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":91.7},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":90.4},{"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score":77.3},{"slug":"mercury-3","name":"Mercury 3","score":75.5}]},{"key":"openHarmonyBench","name":"OpenHarmony Bench","fullName":"OpenHarmony Bench v1.0","category":"coding","description":"An app-level coding benchmark that asks DevEco Code configurations to implement observable behavior in buildable OpenHarmony ArkTS applications.","paperUrl":"https://arxiv.org/abs/2608.16022","paperTitle":"OpenHarmony Bench: Evaluating LLMs and Coding Agents on OpenHarmony App Development","authors":"OpenHarmony Bench authors","year":"2026","tasks":"153 app-development and bug-fix tasks","format":"Task completion through DevEco Code","decimals":1,"modelCount":12,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":60.8},{"slug":"glm-5-3","name":"GLM-5.3","score":60.8},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":60.3},{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":59},{"slug":"glm-5-2","name":"GLM-5.2","score":58.4}]},{"key":"vStar","name":"V*","fullName":"V*","category":"multimodalGrounded","description":"A vision-centric benchmark for high-level multimodal reasoning and perception quality.","paperUrl":"https://docs.z.ai/guides/vlm/glm-5v-turbo","paperTitle":"GLM-5V-Turbo","authors":"Z.AI","year":"2026","tasks":"Frontier multimodal reasoning tasks","format":"Vision-centric reasoning benchmark","decimals":1,"modelCount":12,"lowerIsBetter":false,"top":[{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":96.9},{"slug":"kimi-2-6","name":"Kimi K2.6","score":96.9},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":95.8},{"slug":"step-3-7-flash","name":"Step 3.7 Flash","score":95.3},{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":94.7}]},{"key":"videoMmmu","name":"VideoMMMU","fullName":"VideoMMMU","category":"multimodalGrounded","description":"A video extension of MMMU-style multimodal reasoning over expert questions grounded in temporal media.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Video-grounded expert reasoning","format":"Video + text reasoning","decimals":1,"modelCount":12,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":88.7},{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":87.6},{"slug":"dots3-note-preview","name":"dots3-note Preview","score":86.8},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":86.6},{"slug":"gpt-5-2","name":"GPT-5.2","score":85.9}]},{"key":"cweBenchV1","name":"CWE-bench v1","fullName":"CWE-bench v1","category":"agentic","description":"Defensive vulnerability patching on 120 held-out tasks spanning 73 weakness types.","paperUrl":"https://cwe-bench.com/","paperTitle":"CWE-bench v1 leaderboard and methodology","authors":"Collinear AI","year":"2026","tasks":"Audit and patch held-out C/C++, Go, Java, TypeScript/JavaScript, Python, and Rust repositories","format":"Pass@1","decimals":1,"modelCount":11,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":68},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":68},{"slug":"grok-4-7","name":"Grok 4.7","score":68},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":67},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":58}]},{"key":"mgsm","name":"MGSM","fullName":"Multilingual Grade School Math","category":"multilingual","description":"A multilingual benchmark that translates 250 grade school math problems from GSM8K into 10 typologically diverse languages: Bengali, German, Spanish, French, Japanese, Russian, Swahili, Telugu, Thai, and Chinese.","paperUrl":"https://arxiv.org/abs/2210.03057","paperTitle":"Language Models are Multilingual Chain-of-Thought Reasoners","authors":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","year":"2022","tasks":"250 problems × 11 languages","format":"Math word problems","decimals":null,"modelCount":11,"lowerIsBetter":false,"top":[{"slug":"llama-3-3-70b","name":"Llama 3.3 70B","score":91.1},{"slug":"o1-mini","name":"o1-mini","score":89.9},{"slug":"gpt-4-5","name":"GPT-4.5","score":86.9},{"slug":"llama-3-1-70b","name":"Llama 3.1 70B","score":86.9},{"slug":"claude-3-5-haiku","name":"Claude 3.5 Haiku","score":85.6}]},{"key":"mmmlu","name":"MMMLU","fullName":"MMMLU","category":"knowledge","description":"A multilingual MMLU-style benchmark reported in provider evaluation tables.","paperUrl":"https://huggingface.co/datasets/openai/MMMLU","paperTitle":"MMMLU","authors":"OpenAI","year":"2026","tasks":"Multilingual academic QA","format":"Exact match","decimals":1,"modelCount":11,"lowerIsBetter":false,"top":[{"slug":"interfaze-beta","name":"Interfaze Beta","score":90.9},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":90.3},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":89},{"slug":"claude-4-opus","name":"Claude 4 Opus","score":87.4},{"slug":"k-exaone-2-0","name":"K-EXAONE 2.0","score":86.6}]},{"key":"omniDocBench15","name":"OmniDocBench 1.5","fullName":"OmniDocBench 1.5","category":"multimodalGrounded","description":"A document understanding benchmark used in frontier-model comparison tables to measure extraction and grounded reasoning quality on complex documents.","paperUrl":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","paperTitle":"Introducing GPT-5.4 mini and nano","authors":"OpenAI","year":"2026","tasks":"Document understanding tasks","format":"Document understanding benchmark","decimals":1,"modelCount":11,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":92.1},{"slug":"minimax-m3","name":"MiniMax M3","score":91.6},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":91.4},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":91.1},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":90.8}]},{"key":"vitaBench","name":"VITA-Bench","fullName":"VITA-Bench","category":"agentic","description":"An interactive real-world agent benchmark grounded in practical consumer-service tasks such as delivery, in-store consumption, and online travel workflows.","paperUrl":"https://vitabench.github.io/","paperTitle":"VitaBench: Benchmarking LLM Agents with Versatile Interactive Tasks in Real-world Applications","authors":"Meituan LongCat Team","year":"2025","tasks":"Interactive consumer-service agent tasks","format":"End-to-end interactive agent evaluation","decimals":1,"modelCount":11,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":47.9},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":45.6},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":44.3},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":43.7},{"slug":"agents-a1","name":"Agents-A1","score":38.75}]},{"key":"videoMmeWithSub","name":"Video-MME (with subtitle)","fullName":"Video-MME with subtitle","category":"multimodalGrounded","description":"A video understanding benchmark that allows subtitle access when answering multimodal questions about videos.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Video understanding","format":"Video QA with subtitle context","decimals":1,"modelCount":11,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":90.4},{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":88.4},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":88},{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":87.7},{"slug":"mimo-v2-5","name":"MiMo-V2.5","score":87.7}]},{"key":"cEval","name":"C-Eval","fullName":"C-Eval","category":"knowledge","description":"A Chinese-language academic and professional benchmark spanning humanities, social science, STEM, and applied subjects.","paperUrl":"https://arxiv.org/abs/2305.08322","paperTitle":"C-Eval: A Multi-Level Multi-Discipline Chinese Evaluation Suite for Foundation Models","authors":"C-Eval authors","year":"2023","tasks":"Chinese academic and professional exams","format":"Multiple choice questions","decimals":null,"modelCount":10,"lowerIsBetter":false,"top":[{"slug":"kimi-k2-5","name":"Kimi K2.5","score":94},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":93.3},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":93},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":92.2},{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":91.4}]},{"key":"include","name":"INCLUDE","fullName":"INCLUDE","category":"multilingual","description":"A multilingual benchmark used in provider tables to measure inclusive language coverage and cross-lingual understanding beyond common high-resource languages.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Cross-lingual understanding","format":"Multilingual benchmark","decimals":1,"modelCount":10,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":89.8},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":87.6},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":86.2},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":86.2},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":85.6}]},{"key":"liveCodeBench","name":"LiveCodeBench","fullName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","description":"A continuously updated coding benchmark built from newly collected LeetCode, AtCoder, and Codeforces problems. Fresh problem windows reduce one contamination path, but results still need a release and setup check.","paperUrl":"https://arxiv.org/abs/2403.07974","paperTitle":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","authors":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","year":"2024","tasks":"Continuously updated contest problems","format":"Competitive-programming evaluation","decimals":null,"modelCount":10,"lowerIsBetter":false,"top":[{"slug":"solar-pro-4","name":"Solar Pro 4","score":87.8},{"slug":"deepseek-v3-1-reasoning","name":"DeepSeek V3.1 (Reasoning)","score":74.8},{"slug":"grok-3-mini","name":"Grok 3 Mini","score":74.8},{"slug":"nemotron-ultra-253b","name":"Nemotron Ultra 253B","score":66.31},{"slug":"minimax-m1-80k","name":"MiniMax M1 80k","score":65}]},{"key":"mmAnswerBench","name":"MMAnswerBench","fullName":"MMAnswerBench","category":"math","description":"A multimodal mathematical reasoning benchmark that tests whether models can answer visually grounded math questions correctly.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Multimodal math questions","format":"Visual and structured mathematical QA","decimals":1,"modelCount":10,"lowerIsBetter":false,"top":[{"slug":"glm-5-2","name":"GLM-5.2","score":91},{"slug":"kimi-2-6","name":"Kimi K2.6","score":86},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":84},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":83.8},{"slug":"glm-5-1","name":"GLM-5.1","score":83.8}]},{"key":"qwenClawBench","name":"QwenClawBench","fullName":"QwenClawBench","category":"agentic","description":"Qwen's internal OpenClaw-style benchmark for measuring broad real-world agent performance across practical productivity and research tasks.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Real-world agent workflows","format":"End-to-end agent evaluation","decimals":1,"modelCount":10,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":64.3},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":61.8},{"slug":"qwen3-6-max-preview","name":"Qwen 3.6 Max (preview)","score":59},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":57.2},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":54.3}]},{"key":"simpleQa","name":"SimpleQA","fullName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","description":"A benchmark that evaluates the ability of language models to answer short, fact-seeking questions accurately. Focuses on factual correctness rather than reasoning complexity.","paperUrl":"https://arxiv.org/abs/2411.04368","paperTitle":"Measuring short-form factuality in large language models","authors":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","year":"2024","tasks":"Factual questions","format":"Short-form Q&A","decimals":null,"modelCount":10,"lowerIsBetter":false,"top":[{"slug":"gpt-4-5","name":"GPT-4.5","score":62.5},{"slug":"qwen3-235b-2507","name":"Qwen3 235B 2507","score":54.3},{"slug":"grok-3-beta","name":"Grok 3 [Beta]","score":43.6},{"slug":"kimi-k2","name":"Kimi K2","score":31},{"slug":"deepseek-r1","name":"DeepSeek-R1","score":30.1}]},{"key":"imoAnswerBench","name":"IMOAnswerBench","fullName":"IMOAnswerBench","category":"math","description":"A challenging mathematical reasoning benchmark reported in DeepSeek-V4 model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Advanced mathematical answer generation","format":"Pass@1 math benchmark","decimals":1,"modelCount":9,"lowerIsBetter":false,"top":[{"slug":"dots3-note-preview","name":"dots3-note Preview","score":90.9},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":90},{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":89.8},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":88.4},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":86}]},{"key":"mlvuAvg","name":"MLVU (M-Avg)","fullName":"MLVU mean average","category":"multimodalGrounded","description":"A multi-task video understanding benchmark averaged across MLVU categories.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"General video understanding","format":"Video QA and understanding","decimals":1,"modelCount":9,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":90.8},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":87.4},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":86.7},{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":86.6},{"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score":86.2}]},{"key":"mmmuProPython","name":"MMMU-Pro w/ Python","fullName":"MMMU-Pro with Python","category":"multimodalGrounded","description":"Tool-augmented MMMU-Pro variant that allows Python assistance during multimodal reasoning.","paperUrl":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","paperTitle":"Introducing GPT-5.4 mini and nano","authors":"OpenAI","year":"2026","tasks":"Multimodal academic reasoning","format":"Image + text question answering with Python","decimals":null,"modelCount":9,"lowerIsBetter":false,"top":[{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":84.6},{"slug":"kimi-k3","name":"Kimi K3","score":83.4},{"slug":"gpt-5-5","name":"GPT-5.5","score":83.2},{"slug":"gpt-5-4","name":"GPT-5.4","score":82.1},{"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","score":82}]},{"key":"stemBench120","name":"STEMBench","fullName":"STEMBench120: six-domain scientific research progress","category":"reasoning","description":"Scores how far a model advances 120 open research tasks across biology, chemistry, physics, astronomy, Earth science and engineering on a 0 to 4 progress scale, crediting supported partial work and honest statements of what remains unresolved.","paperUrl":"https://www.ulam.ai/stembench","paperTitle":"STEMBench: six-domain model evaluation","authors":"Ulam","year":"2026","tasks":"120 research tasks, 20 in each of six STEM domains","format":"Audited progress level 0 to 4 per task, refined to a fractional score; complete-run mean across all tasks","decimals":2,"modelCount":9,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":2.809},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":2.759},{"slug":"muse-wonder","name":"Muse Wonder","score":2.683},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":2.621},{"slug":"hy4-preview","name":"Hy4 preview","score":2.586}]},{"key":"apexAgents","name":"APEX-Agents","fullName":"APEX-Agents","category":"agentic","description":"A professional-services agent benchmark covering long-horizon knowledge-work tasks.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI / APEX-Agents benchmark authors","year":"2026","tasks":"Professional-services agent tasks","format":"Agent task-completion score","decimals":1,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"grok-4-6","name":"Grok 4.6","score":57.5},{"slug":"apodex-1-1","name":"Apodex 1.1","score":38.5},{"slug":"step-5-preview","name":"Step 5 Preview","score":37.8},{"slug":"kimi-k3","name":"Kimi K3","score":37.6},{"slug":"hy4-preview","name":"Hy4 preview","score":37.1}]},{"key":"arcAgi3ProviderAdapter","name":"ARC-AGI-3 Provider Adapter","fullName":"ARC-AGI-3 Semi-Private: Provider Adapter harness","category":"reasoning","description":"ARC Prize verified interactive reasoning results using a provider-specific stateful adapter, kept separate from Standard harness results.","paperUrl":"https://arcprize.org/leaderboard","paperTitle":"ARC Prize verified ARC-AGI-3 results","authors":"ARC Prize Foundation","year":2026,"tasks":null,"format":null,"decimals":2,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":98.550743},{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":97.304327},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":96.177391},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":35.004243},{"slug":"gpt-6-sol","name":"GPT-6 Sol","score":23.028057}]},{"key":"androidWorld","name":"AndroidWorld","fullName":"AndroidWorld","category":"agentic","description":"A mobile GUI agent benchmark for completing Android app workflows and on-device tasks.","paperUrl":"https://docs.z.ai/guides/vlm/glm-5v-turbo","paperTitle":"GLM-5V-Turbo","authors":"Z.AI","year":"2026","tasks":"Android app workflows","format":"Interactive mobile-agent evaluation","decimals":1,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":87.1},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":85.3},{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":84.5},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":81.9},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":81}]},{"key":"apex","name":"Apex","fullName":"Apex","category":"math","description":"A high-difficulty mathematical reasoning benchmark reported in DeepSeek-V4 model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Advanced mathematical reasoning","format":"Pass@1 math benchmark","decimals":1,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"hy4-preview","name":"Hy4 preview","score":74.2},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":65.6},{"slug":"a-x-k2","name":"A.X K2","score":45.8},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":44.5},{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":38.3}]},{"key":"babyVision","name":"BabyVision","fullName":"BabyVision","category":"multimodalGrounded","description":"A multimodal benchmark for fine-grained visual perception and grounded reasoning tasks.","paperUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","paperTitle":"Muse Spark 1.1 Evaluation Report","authors":"Meta AI","year":"2026","tasks":"Visual perception tasks","format":"Multimodal visual reasoning","decimals":1,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":82},{"slug":"muse-spark-open","name":"Muse Spark Open","score":80.2},{"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score":76.3},{"slug":"seed-2-1-pro","name":"Seed 2.1 Pro","score":73.7},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":65.7}]},{"key":"countBench","name":"CountBench","fullName":"CountBench","category":"multimodalGrounded","description":"A visual counting benchmark that tests whether a model can count objects and entities reliably in complex scenes.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Visual counting tasks","format":"Image-grounded counting","decimals":1,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":97.8},{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":97.3},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":97.2},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":94.1},{"slug":"gpt-5-2","name":"GPT-5.2","score":91.9}]},{"key":"frontierCode11Extended","name":"FrontierCode 1.1 Extended","fullName":"FrontierCode 1.1 Extended","category":"coding","description":"Cognition's 150-task Extended subset of the FrontierCode 1.1 software-engineering benchmark.","paperUrl":"https://cognition.com/frontiercode","paperTitle":"FrontierCode leaderboard","authors":"Cognition","year":"2026","tasks":"150 private software-engineering tasks","format":"Repository task completion with maintainer rubrics","decimals":1,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":64.5},{"slug":"claude-opus-5","name":"Claude Opus 5","score":63.6},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":63.6},{"slug":"grok-4-6","name":"Grok 4.6","score":61.3},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":60.6}]},{"key":"medXpertQaMm","name":"MedXpertQA (MM)","fullName":"MedXpertQA Multimodal","category":"multimodalGrounded","description":"A multimodal medical multiple-choice benchmark covering clinical images such as X-rays, histology, and dermatology.","paperUrl":"https://ai.meta.com/static-resource/muse-spark-eval-methodology","paperTitle":"Muse Spark Eval Methodology","authors":"Meta AI","year":"2026","tasks":"2,000 multimodal medical questions","format":"Medical visual MCQ","decimals":1,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score":81.3},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":80.4},{"slug":"muse-spark","name":"Muse Spark","score":78.4},{"slug":"gpt-5-4","name":"GPT-5.4","score":77.1},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":71}]},{"key":"refcocoAvg","name":"RefCOCO (avg)","fullName":"RefCOCO average","category":"multimodalGrounded","description":"A referring-expression grounding benchmark averaged across RefCOCO variants to test whether a model can localize described objects correctly.","paperUrl":"https://github.com/lichengunc/refer","paperTitle":"RefCOCO referring expression datasets","authors":"RefCOCO dataset authors","year":"2026","tasks":"Referring-expression grounding","format":"Grounded visual localization","decimals":1,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":92.5},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":92.3},{"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score":92},{"slug":"nemotron-3-nano-omni-30b-a3b","name":"Nemotron 3 Nano Omni 30B A3B","score":90.5},{"slug":"lfm2-5-vl-3b","name":"LFM2.5-VL-3B","score":87.9}]},{"key":"zeroBench","name":"ZeroBench","fullName":"ZeroBench","category":"multimodalGrounded","description":"A multi-step visual reasoning benchmark with pass@5 reporting and optional tool use.","paperUrl":"https://ai.meta.com/static-resource/muse-spark-eval-methodology","paperTitle":"Muse Spark Eval Methodology","authors":"Meta AI","year":"2026","tasks":"100 visual reasoning questions","format":"Multi-step visual reasoning","decimals":1,"modelCount":8,"lowerIsBetter":false,"top":[{"slug":"muse-spark-open","name":"Muse Spark Open","score":50},{"slug":"gpt-5-4","name":"GPT-5.4","score":41},{"slug":"muse-spark","name":"Muse Spark","score":33},{"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score":29},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":24}]},{"key":"ai2dTest","name":"AI2D_TEST","fullName":"AI2D test split","category":"multimodalGrounded","description":"A diagram understanding benchmark focused on scientific and educational visual question answering.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Diagram understanding","format":"Diagram-grounded QA","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":94.1},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":93.9},{"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score":92.7},{"slug":"gpt-5-2","name":"GPT-5.2","score":92.2},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":90.8}]},{"key":"bbh","name":"BBH","fullName":"BIG-Bench Hard","category":"reasoning","description":"A suite of 23 challenging tasks from the BIG-Bench collaborative benchmark where prior language models failed to exceed average human performance, even with chain-of-thought prompting.","paperUrl":"https://arxiv.org/abs/2210.09261","paperTitle":"Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them","authors":"Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, Jason Wei","year":"2022","tasks":"23 tasks","format":"Mixed reasoning tasks","decimals":null,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"claude-3-5-haiku","name":"Claude 3.5 Haiku","score":86.6},{"slug":"soofi-s-30b-a3b","name":"Soofi S 30B-A3B","score":78.8},{"slug":"minicpm5-1b","name":"MiniCPM5-1B","score":71.89},{"slug":"phi-3-5-mini","name":"Phi-3.5-mini","score":69},{"slug":"granite-4-0-h-1b","name":"Granite-4.0-H-1B","score":59.68}]},{"key":"ccOcr","name":"CC-OCR","fullName":"CC-OCR","category":"multimodalGrounded","description":"An OCR-focused benchmark for reading and extracting text from visually complex documents and images.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Optical character recognition","format":"Text extraction from images and documents","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":82},{"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score":81.9},{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":81.2},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":79.7},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":79.6}]},{"key":"deepPlanning","name":"DeepPlanning","fullName":"DeepPlanning","category":"agentic","description":"A long-horizon planning benchmark that tests whether agents can optimize under explicit time, budget, and feasibility constraints.","paperUrl":"https://arxiv.org/abs/2601.18137","paperTitle":"DeepPlanning: Benchmarking Long-Horizon Agentic Planning with Verifiable Constraints","authors":"DeepPlanning authors","year":"2026","tasks":"Travel planning and constrained shopping","format":"Long-horizon planning benchmark","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":62.3},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":41.5},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":37.6},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":26.4},{"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score":25.9}]},{"key":"frontierMath","name":"FrontierMath (legacy)","fullName":"FrontierMath legacy aggregate","category":"math","description":"Legacy FrontierMath values retained for historical model pages. This field is not used in current rankings because it can mix prior benchmark versions and slices.","paperUrl":"https://epoch.ai/frontiermath","paperTitle":"FrontierMath: A Benchmark for Evaluating Advanced Mathematical Reasoning in AI","authors":"Epoch AI","year":"2024","tasks":"Historical aggregate","format":"Open-ended mathematical reasoning with tool access","decimals":null,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":89},{"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","score":84.9},{"slug":"gpt-5-6-luna","name":"GPT-5.6 Luna","score":78.6},{"slug":"gpt-5-5-pro","name":"GPT-5.5 Pro","score":52.4},{"slug":"gpt-5-5","name":"GPT-5.5","score":51.7}]},{"key":"healthBenchLengthAdjusted","name":"HealthBench (length-adjusted)","fullName":"HealthBench length-adjusted score","category":"knowledge","description":"HealthBench score after applying a verbosity penalty to model responses.","paperUrl":"https://arxiv.org/abs/2505.08775","paperTitle":"HealthBench: Evaluating Large Language Models Towards Improved Human Health","authors":"OpenAI","year":"2025","tasks":"5,000 multi-turn patient conversations","format":"Length-adjusted rubric score","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":65.4},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":60.6},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":58.5},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":58.3},{"slug":"claude-opus-5","name":"Claude Opus 5","score":57.8}]},{"key":"healthBench","name":"HealthBench (raw)","fullName":"HealthBench raw score","category":"knowledge","description":"Raw score on realistic multi-turn healthcare conversations graded against expert-written rubrics.","paperUrl":"https://arxiv.org/abs/2505.08775","paperTitle":"HealthBench: Evaluating Large Language Models Towards Improved Human Health","authors":"OpenAI","year":"2025","tasks":"5,000 multi-turn patient conversations","format":"Raw rubric score","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":69.4},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":68.1},{"slug":"claude-opus-5","name":"Claude Opus 5","score":67.1},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":56.9},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":56.7}]},{"key":"healthBenchProfessionalRaw","name":"HealthBench Professional (raw)","fullName":"HealthBench Professional raw score","category":"knowledge","description":"Raw score on physician-authored clinical consult, documentation, and research conversations.","paperUrl":"https://arxiv.org/abs/2604.27470","paperTitle":"HealthBench Professional: Evaluating Large Language Models on Real Clinician Chats","authors":"OpenAI","year":"2026","tasks":"525 physician-authored conversations","format":"Raw rubric score","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":77.1},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":77.1},{"slug":"claude-opus-5","name":"Claude Opus 5","score":73.4},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":68.2},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":67.2}]},{"key":"labBench2","name":"LABBench2","fullName":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","category":"knowledge","description":"A benchmark of realistic biology-research tasks involving literature, figures, tables, databases, and bioinformatics files.","paperUrl":"https://arxiv.org/abs/2604.09554","paperTitle":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","authors":"Jon M. Laurent, Albert Bou, Michael Pieler, Conor Igoe, Alex Andonian, Siddharth Narayanan, James Braza, Alexandros Sanchez Vassopoulos, Jacob L. Steenwyk, Blake Lash, Andrew D. White, Samuel G. Rodriques","year":"2026","tasks":"Nearly 1,900 biology-research tasks","format":"Aggregate accuracy","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":88.8},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":86.2},{"slug":"claude-opus-5","name":"Claude Opus 5","score":84.2},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":82.1},{"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","score":82.1}]},{"key":"mmvu","name":"MMVU","fullName":"Multimodal Multi-disciplinary Video Understanding","category":"multimodalGrounded","description":"A benchmark for evaluating multimodal models on video understanding tasks across multiple disciplines, emphasizing temporal reasoning and comprehension over video content.","paperUrl":"https://www.kimi.com/blog/kimi-k2-5.html","paperTitle":"Kimi K2.5 benchmark release surface","authors":"MMVU benchmark maintainers","year":"2026","tasks":"Video understanding","format":"Video reasoning benchmark","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":82.4},{"slug":"glm-5-3-flash","name":"GLM-5.3-Flash","score":80.5},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":80.4},{"slug":"dots3-note-preview","name":"dots3-note Preview","score":79.9},{"slug":"qwen3-5-122b-a10b","name":"Qwen3.5-122B-A10B","score":74.7}]},{"key":"nova63","name":"NOVA-63","fullName":"NOVA-63","category":"multilingual","description":"A broad multilingual benchmark row from Qwen's launch comparisons intended to measure cross-lingual capability beyond a single language family.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Broad multilingual evaluation","format":"Cross-lingual benchmark","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":59.1},{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":59},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":58.8},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":57.9},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":56.7}]},{"key":"pinchBench","name":"PinchBench","fullName":"PinchBench","category":"agentic","description":"An OpenClaw agent benchmark from Kilo that measures successful task completion across standardized real-world agent workflows.","paperUrl":"https://pinchbench.com/about","paperTitle":"About PinchBench","authors":"Kilo Code","year":"2026","tasks":"23 OpenClaw agent tasks","format":"Average success rate from official runs","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"pokee-isaac-28b","name":"Pokee-Isaac 28B","score":95.67},{"slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","score":90},{"slug":"nemotron-3-5-bolt-74b-a9b","name":"Nemotron 3.5 Bolt 74B-A9B","score":87.82},{"slug":"nemotron-3-5-lightning-30b-a3b-nvfp4","name":"Nemotron 3.5 Lightning 30B A3B NVFP4","score":83.43},{"slug":"lfm2-5-24b-a2b","name":"LFM2.5-24B-A2B","score":83.18}]},{"key":"sweMultimodal","name":"SWE Multimodal","fullName":"SWE-bench Multimodal","category":"coding","description":"A multimodal variant of SWE-bench that adds visual context such as screenshots and design mockups to software engineering issue descriptions.","paperUrl":"https://www.swebench.com/multimodal","paperTitle":"SWE-bench Multimodal","authors":"SWE-bench team","year":"2025","tasks":"Multimodal software engineering tasks","format":"Code patch generation with visual context","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":61.4},{"slug":"claude-opus-5","name":"Claude Opus 5","score":59.4},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":54.9},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":54.7},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":54.3}]},{"key":"sweMultimodal","name":"SWE-bench Multimodal","fullName":"SWE-bench Multimodal","category":"multimodalGrounded","description":"A multimodal variant of SWE-bench that adds visual context (screenshots, design mockups) to software engineering issue descriptions, testing whether models can leverage visual information for code generation.","paperUrl":"https://www.swebench.com/multimodal","paperTitle":"SWE-bench Multimodal","authors":"SWE-bench team","year":"2025","tasks":"Multimodal software engineering tasks","format":"Code patch generation with visual context","decimals":null,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":61.4},{"slug":"claude-opus-5","name":"Claude Opus 5","score":59.4},{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":54.9},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":54.7},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":54.3}]},{"key":"videoMmeNoSub","name":"Video-MME (w/o subtitle)","fullName":"Video-MME without subtitle","category":"multimodalGrounded","description":"A stricter Video-MME setting that removes subtitle help and tests video understanding from visual and audio context alone.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Video understanding","format":"Video QA without subtitle context","decimals":1,"modelCount":7,"lowerIsBetter":false,"top":[{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":87.7},{"slug":"gpt-5-2","name":"GPT-5.2","score":85.8},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":83.7},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":83.2},{"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score":82.5}]},{"key":"bioMysteryBenchHumanDifficult","name":"BioMysteryBench (human-difficult)","fullName":"BioMysteryBench Human Difficult","category":"knowledge","description":"Computational biology challenges with objective answers that remained unsolved by independent human experts.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"Human-difficult computational biology investigations","format":"Task score","decimals":1,"modelCount":6,"lowerIsBetter":false,"top":[{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":56.5},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":50},{"slug":"claude-opus-5","name":"Claude Opus 5","score":49.4},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":44.7},{"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","score":43.5}]},{"key":"draco","name":"DRACO","fullName":"Data Research and Analysis with Complex Operations","category":"agentic","description":"Agentic data-analysis tasks scored against per-task rubrics at a 980K-token budget.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"Agentic data research and analysis tasks","format":"Normalized rubric score","decimals":1,"modelCount":6,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":88.6},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":87},{"slug":"ling-3-1-flash","name":"Ling 3.1 Flash","score":85.49},{"slug":"step-5-preview","name":"Step 5 Preview","score":83.3},{"slug":"hy4-preview","name":"Hy4 preview","score":77.2}]},{"key":"dynaMath","name":"DynaMath","fullName":"DynaMath","category":"multimodalGrounded","description":"A multimodal benchmark for dynamic mathematical reasoning over visual and structured inputs.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Dynamic visual math problems","format":"Multimodal mathematical reasoning","decimals":1,"modelCount":6,"lowerIsBetter":false,"top":[{"slug":"gpt-5-2","name":"GPT-5.2","score":86.8},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":86.3},{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":85.6},{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":85.1},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":84.4}]},{"key":"gsm8k","name":"GSM8K","fullName":"Grade School Math 8K","category":"math","description":"GSM8K tests multi-step arithmetic in grade-school word problems. OpenAI introduced the dataset in 2021; a high score on this fixed test is one piece of evidence about mathematical reasoning.","paperUrl":"https://arxiv.org/abs/2110.14168","paperTitle":"Training Verifiers to Solve Math Word Problems","authors":"Karl Cobbe and colleagues (OpenAI)","year":"2021","tasks":"Grade-school math word problems","format":"Exact match","decimals":1,"modelCount":6,"lowerIsBetter":false,"top":[{"slug":"ternary-bonsai-2-27b","name":"Ternary Bonsai 2 27B","score":96.66},{"slug":"llama-3-1-70b","name":"Llama 3.1 70B","score":95.1},{"slug":"celeris-1","name":"Celeris-1","score":93.7},{"slug":"phi-3-5-mini","name":"Phi-3.5-mini","score":86.2},{"slug":"soofi-s-30b-a3b","name":"Soofi S 30B-A3B","score":86.1}]},{"key":"hleVerified","name":"HLE-Verified","fullName":"HLE-Verified","category":"knowledge","description":"A verified and revised Humanity's Last Exam set that removes uncertain items and repairs fixable questions before evaluating frontier expert reasoning.","paperUrl":"https://arxiv.org/abs/2602.13964","paperTitle":"HLE-Verified: A Systematic Verification and Structured Revision of Humanity's Last Exam","authors":"Weiqi Zhai, Zhihai Wang, Jinghang Wang, Boyu Yang, Xiaogang Li, Xiang Xu, Bohan Wang, Peng Wang, Xingzhe Wu, Anfeng Li, Qiyuan Feng, Yuhao Zhou, Shoulin Han, Wenjie Luo, Yiyuan Li, Yaxuan Wang, Ruixian Luo, Guojie Lin, Peiyao Xiao, Chengliang Xu, Ben Wang, Zeyu Wang, Zichao Chen, Jianan Ye, Yijie Hu, Jialong Chen, Zongwen Shen, Yuliang Xu, An Yang, Bowen Yu, Dayiheng Liu, Junyang Lin, Hu Wei, Que Shen, Bing Zhao","year":"2026","tasks":"1,811 verified or revised expert questions","format":"Full-set accuracy","decimals":1,"modelCount":6,"lowerIsBetter":false,"top":[{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":54.9},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":54.5},{"slug":"claude-opus-5","name":"Claude Opus 5","score":54.4},{"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","score":53.6},{"slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","score":51.1}]},{"key":"lvBench","name":"LVBench","fullName":"LVBench","category":"multimodalGrounded","description":"A long-video understanding benchmark for retrieving and reasoning over information distributed across extended video inputs.","paperUrl":"https://qwen.ai/blog?id=qwen3.8","paperTitle":"Qwen3.8-Max: A New Bar for Coding and Cowork","authors":"Qwen Team","year":"2026","tasks":"Long-form video question answering","format":"Long-video understanding score","decimals":1,"modelCount":6,"lowerIsBetter":false,"top":[{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":91.7},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":87.1},{"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","score":85.4},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":81.8},{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":76.9}]},{"key":"liveCodeBenchPro","name":"LiveCodeBench Pro","fullName":"LiveCodeBench Pro","category":"coding","description":"A harder competitive-programming benchmark family built from Codeforces, ICPC, and IOI problems, with quarter-specific public leaderboards and difficulty-aware reporting.","paperUrl":"https://arxiv.org/abs/2506.11928","paperTitle":"LiveCodeBench Pro: How Do Olympiad Medalists Judge LLMs in Competitive Programming?","authors":"LiveCodeBench Pro authors","year":"2025","tasks":"Quarter-specific contest programming sets","format":"Competitive programming","decimals":1,"modelCount":6,"lowerIsBetter":false,"top":[{"slug":"gpt-5-4","name":"GPT-5.4","score":87.5},{"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score":82.9},{"slug":"muse-spark","name":"Muse Spark","score":80},{"slug":"grok-4-20-beta","name":"Grok 4.20","score":74.2},{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score":70.7}]},{"key":"liveCodeBenchV5","name":"LiveCodeBench v5","fullName":"LiveCodeBench v5","category":"coding","description":"LiveCodeBench v5 is a named release and date-window slice. BenchLM keeps explicitly labeled v5 rows outside the rolling weighted lane.","paperUrl":"https://github.com/LiveCodeBench/LiveCodeBench","paperTitle":"LiveCodeBench official repository and release documentation","authors":"LiveCodeBench maintainers","year":"2025","tasks":"July 2024 to May 2025 release window","format":"Provider-published v5 competitive programming result","decimals":1,"modelCount":6,"lowerIsBetter":false,"top":[{"slug":"ling-3-0-flash","name":"Ling 3.0 Flash","score":82.8},{"slug":"nemotron-3-nano-omni-30b-a3b","name":"Nemotron 3 Nano Omni 30B A3B","score":63.2},{"slug":"qwq-32b","name":"QwQ-32B","score":62.7},{"slug":"magistral-medium","name":"Magistral Medium","score":59.4},{"slug":"qwen3-235b-a22b","name":"Qwen3 235B-A22B","score":35.3}]},{"key":"polyMath","name":"PolyMath","fullName":"PolyMath","category":"multilingual","description":"A multilingual mathematical reasoning benchmark that tests whether math performance transfers across languages rather than only in English.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Multilingual math problems","format":"Cross-lingual mathematical reasoning","decimals":1,"modelCount":6,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":86.5},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":84},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":79},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":73.3},{"slug":"k-exaone-2-0","name":"K-EXAONE 2.0","score":71.3}]},{"key":"aime2025Arcee","name":"AIME25 (Arcee)","fullName":"AIME25 first-party comparison snapshot","category":"math","description":"A display-only AIME25 reference from Arcee AI's Trinity-Large-Thinking launch chart.","paperUrl":"https://www.arcee.ai/blog/trinity-large-thinking","paperTitle":"Trinity-Large-Thinking: Scaling an Open Source Frontier Agent","authors":"Arcee AI","year":"2026","tasks":"15 problems","format":"Integer answers 000-999","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score":99.8},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":96.3},{"slug":"trinity-large-thinking","name":"Trinity-Large-Thinking","score":96.3},{"slug":"glm-5","name":"GLM-5","score":93.3},{"slug":"minimax-m2-7","name":"MiniMax M2.7","score":80}]},{"key":"bfclV3","name":"BFCL v3","fullName":"Berkeley Function Calling Leaderboard v3","category":"agentic","description":"A function-calling benchmark for tool selection, schema adherence, and argument correctness, covering single-turn, parallel, irrelevance and multi-turn subsets.","paperUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","paperTitle":"The Berkeley Function Calling Leaderboard (BFCL): From Tool Use to Agentic Evaluation of Large Language Models","authors":"Shishir G. Patil et al.","year":"2025","tasks":"Function-calling tasks","format":"Tool invocation and schema evaluation","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"ternary-bonsai-2-27b","name":"Ternary Bonsai 2 27B","score":74.92},{"slug":"qwen3-coder-480b-a35b","name":"Qwen3-Coder 480B-A35B","score":68.7},{"slug":"qwen3-235b-a22b","name":"Qwen3 235B-A22B","score":68},{"slug":"qwq-32b","name":"QwQ-32B","score":66.4},{"slug":"qwen3-32b","name":"Qwen3 32B","score":63}]},{"key":"babyVisionPython","name":"BabyVision w/ Python","fullName":"BabyVision with Python","category":"multimodalGrounded","description":"A Python-assisted BabyVision evaluation for fine-grained visual perception and grounded reasoning.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI","year":"2026","tasks":"Visual perception tasks with Python","format":"Tool-augmented multimodal score","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":92.5},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":91.3},{"slug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","score":91},{"slug":"kimi-k3","name":"Kimi K3","score":85.7},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":85.6}]},{"key":"bioMysteryBenchHumanSolvable","name":"BioMysteryBench (human-solvable)","fullName":"BioMysteryBench Human Solvable","category":"knowledge","description":"Computational biology challenges that independent human experts were able to solve.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"Human-solvable computational biology investigations","format":"Task score","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":90.1},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":89.3},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":89.2},{"slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","score":88.8},{"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","score":87.1}]},{"key":"chartography","name":"Chartography (no tools)","fullName":"Chartography without tools","category":"multimodalGrounded","description":"Professional chart understanding across 100 specialized chart types with expert-set answer tolerances.","paperUrl":"https://surgehq.ai/blog/chartography","paperTitle":"Chartography","authors":"Surge AI and Anthropic","year":"2026","tasks":"100 specialized chart types","format":"Accuracy without tools","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":71.6},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":64.4},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":61.6},{"slug":"claude-haiku-5-5","name":"Claude Haiku 5.5","score":46.4},{"slug":"claude-opus-5","name":"Claude Opus 5","score":29.6}]},{"key":"chartographyWithTools","name":"Chartography (tools)","fullName":"Chartography with image and code tools","category":"multimodalGrounded","description":"Professional chart understanding with a container, standard libraries, and image cropping.","paperUrl":"https://surgehq.ai/blog/chartography","paperTitle":"Chartography","authors":"Surge AI and Anthropic","year":"2026","tasks":"100 specialized chart types","format":"Accuracy with tools","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":90.2},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":89},{"slug":"claude-opus-5","name":"Claude Opus 5","score":83},{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":83},{"slug":"glm-5-3-flash","name":"GLM-5.3-Flash","score":78}]},{"key":"drop","name":"DROP","fullName":"Discrete Reasoning Over Paragraphs","category":"reasoning","description":"A reading-comprehension benchmark requiring discrete reasoning over paragraphs, reported in DeepSeek-V4 base-model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Paragraph reasoning questions","format":"F1","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"o1-mini","name":"o1-mini","score":83.9},{"slug":"gpt-4-5","name":"GPT-4.5","score":83.4},{"slug":"claude-3-5-haiku","name":"Claude 3.5 Haiku","score":83.1},{"slug":"celeris-1","name":"Celeris-1","score":81.4},{"slug":"soofi-s-30b-a3b","name":"Soofi S 30B-A3B","score":66.5}]},{"key":"extractBenchPro","name":"ExtractBench Pro","fullName":"ExtractBench Pro: long-tail document extraction challenges (LlamaIndex)","category":"multimodalGrounded","description":"A harder companion to ExtractBench: 100 synthetic documents and 2,622 pages across 86 document types that test document conventions, house vocabulary, counting rules, drawn relations and charts, scored by perfect-document rate.","paperUrl":"https://www.extractbench.ai/pro","paperTitle":"ExtractBench Pro leaderboard and challenge descriptions","authors":"LlamaIndex","year":"2026","tasks":"100 synthetic documents, 2,622 pages, 86 document types, six challenge groups","format":"Perfect-document rate, value F1, word and page grounding F1 and cost, overall and per challenge","decimals":0,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":58},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":51},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":32},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":12},{"slug":"gpt-6-luna","name":"GPT-6 Luna","score":0}]},{"key":"maxife","name":"MAXIFE","fullName":"MAXIFE","category":"multilingual","description":"A multilingual instruction-following and understanding benchmark row published in Qwen's launch comparisons.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Multilingual instruction following","format":"Cross-lingual benchmark","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":89.2},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":88.8},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":88.2},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":79.2},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":72.8}]},{"key":"mcpTasks","name":"MCP-Tasks","fullName":"MCP-Tasks","category":"agentic","description":"A Model Context Protocol task benchmark used in Qwen's launch tables to measure practical execution over MCP-style tools and integrations.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"MCP-integrated tool tasks","format":"Interactive tool-use evaluation","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":74.2},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":74.1},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":71.8},{"slug":"glm-5","name":"GLM-5","score":60.8},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":59.1}]},{"key":"mmluProArcee","name":"MMLU-Pro (Arcee)","fullName":"MMLU-Pro first-party comparison snapshot","category":"knowledge","description":"A display-only MMLU-Pro reference from Arcee AI's Trinity-Large-Thinking launch chart.","paperUrl":"https://www.arcee.ai/blog/trinity-large-thinking","paperTitle":"Trinity-Large-Thinking: Scaling an Open Source Frontier Agent","authors":"Arcee AI","year":"2026","tasks":"Professional academic QA","format":"10-way multiple choice","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score":89.1},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":87.1},{"slug":"glm-5","name":"GLM-5","score":85.8},{"slug":"trinity-large-thinking","name":"Trinity-Large-Thinking","score":83.4},{"slug":"minimax-m2-7","name":"MiniMax M2.7","score":80.8}]},{"key":"mmLongBenchDoc","name":"MMLongBench-Doc","fullName":"MMLongBench-Doc","category":"multimodalGrounded","description":"A long-document multimodal benchmark for grounded reasoning over extended document contexts.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Long document understanding","format":"Document-grounded reasoning","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"nemotron-3-5-super-vl-120b-a12b","name":"Nemotron 3.5 Super VL 120B A12B","score":62.82},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":61.5},{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":60.5},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":58.5},{"slug":"nemotron-3-nano-omni-30b-a3b","name":"Nemotron 3 Nano Omni 30B A3B","score":57.5}]},{"key":"mathVisionPython","name":"MathVision w/ Python","fullName":"MathVision with Python","category":"multimodalGrounded","description":"A tool-augmented MathVision variant that permits Python during visual mathematics reasoning.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI / MathVision authors","year":"2026","tasks":"Visual mathematics problems with Python","format":"Image and mathematics reasoning with tools","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"kimi-k3","name":"Kimi K3","score":97.8},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":97.7},{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":96.2},{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":95.7},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":94.6}]},{"key":"medXpertQaText","name":"MedXpertQA (Text)","fullName":"MedXpertQA Text","category":"knowledge","description":"A medical multiple-choice benchmark spanning many specialties with 10 answer options per question.","paperUrl":"https://ai.meta.com/static-resource/muse-spark-eval-methodology","paperTitle":"Muse Spark Eval Methodology","authors":"Meta AI","year":"2026","tasks":"2,450 medical multiple-choice questions","format":"Medical MCQ","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","score":71.5},{"slug":"gpt-5-4","name":"GPT-5.4","score":59.6},{"slug":"muse-spark","name":"Muse Spark","score":52.6},{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score":52.1},{"slug":"grok-4-20-beta","name":"Grok 4.20","score":50.2}]},{"key":"ocrBenchV2","name":"OCRBench V2","fullName":"OCRBench V2","category":"multimodalGrounded","description":"A native OCR benchmark for reading text from images across multilingual scripts, low-quality scans, handwriting, structured layouts, charts, and screenshots.","paperUrl":"https://arxiv.org/abs/2501.00321","paperTitle":"OCRBench v2: An Improved Benchmark for Evaluating Large Multimodal Models on Visual Text Localization and Reasoning","authors":"OCRBench authors","year":"2025","tasks":"Image OCR tasks","format":"Accuracy","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":74.2},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":70.7},{"slug":"interfaze-beta","name":"Interfaze Beta","score":70.7},{"slug":"ternary-bonsai-2-27b","name":"Ternary Bonsai 2 27B","score":56.88},{"slug":"lfm2-5-vl-3b","name":"LFM2.5-VL-3B","score":47.5}]},{"key":"qwenWebBench","name":"QwenWebBench","fullName":"QwenWebBench","category":"agentic","description":"A Qwen benchmark for artifact and webpage generation quality reported as an Elo-style rating.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Web artifacts and interactive deliverables","format":"Elo-style artifact benchmark","decimals":0,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-max","name":"Qwen3.7 Max","score":1568},{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":1536},{"slug":"qwen3-6-max-preview","name":"Qwen 3.6 Max (preview)","score":1532},{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":1487},{"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score":1397}]},{"key":"sweVerifiedArcee","name":"SWE-bench Verified*","fullName":"SWE-bench Verified (mini-swe-agent-v2)","category":"coding","description":"A display-only SWE-bench Verified reference from Arcee AI's Trinity-Large-Thinking comparison chart.","paperUrl":"https://www.arcee.ai/blog/trinity-large-thinking","paperTitle":"Trinity-Large-Thinking: Scaling an Open Source Frontier Agent","authors":"Arcee AI","year":"2026","tasks":"Repository task completion","format":"Agent scaffold benchmark","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-6","name":"Claude Opus 4.6","score":75.6},{"slug":"minimax-m2-7","name":"MiniMax M2.7","score":75.4},{"slug":"glm-5","name":"GLM-5","score":72.8},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":70.8},{"slug":"trinity-large-thinking","name":"Trinity-Large-Thinking","score":63.2}]},{"key":"terminalBenchScience","name":"Terminal-Bench-Science 0.1","fullName":"Terminal-Bench-Science 0.1","category":"agentic","description":"A benchmark of AI agents completing expert-curated research workflows across the life, physical, Earth, mathematical, and engineering sciences.","paperUrl":"https://www.terminal-bench-science.ai/","paperTitle":"Terminal-Bench-Science 0.1","authors":"Terminal-Bench-Science Team","year":"2026","tasks":"70 expert-curated scientific research workflows","format":"Resolution rate across 3 trials per task","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":64.6},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":59.9},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":58.7},{"slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","score":57},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":52.6}]},{"key":"weMath","name":"We-Math","fullName":"We-Math","category":"multimodalGrounded","description":"A multimodal math benchmark for visually grounded mathematical reasoning and answer generation.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Visually grounded math problems","format":"Multimodal mathematical reasoning","decimals":1,"modelCount":5,"lowerIsBetter":false,"top":[{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":87.9},{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":86.9},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":84.7},{"slug":"gpt-5-2","name":"GPT-5.2","score":79},{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":70}]},{"key":"aiNeedle","name":"AI-Needle","fullName":"AI-Needle","category":"reasoning","description":"A long-context retrieval benchmark that measures whether a model can recover relevant information embedded deep inside very long contexts.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Long-context retrieval","format":"Needle-in-a-haystack recall","decimals":null,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":74},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":68.7},{"slug":"qwen3-6-plus","name":"Qwen3.6 Plus","score":68.3},{"slug":"glm-5","name":"GLM-5","score":63.3}]},{"key":"apexShortlist","name":"Apex Shortlist","fullName":"Apex Shortlist","category":"math","description":"A shortlist subset of the Apex mathematical reasoning benchmark reported in DeepSeek-V4 model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Advanced mathematical reasoning","format":"Pass@1 math benchmark","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":90.2},{"slug":"a-x-k2","name":"A.X K2","score":88.6},{"slug":"nemotron-3-5-super-vl-120b-a12b","name":"Nemotron 3.5 Super VL 120B A12B","score":88.36},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":85.7}]},{"key":"benchCadVision2CodeWithTools","name":"BenchCAD Vision2Code (tools)","fullName":"BenchCAD Vision2Code voxel IoU with tools","category":"multimodalGrounded","description":"Generates CadQuery code from multi-view renders with image inspection and code-execution tools.","paperUrl":"https://arxiv.org/abs/2605.10865","paperTitle":"BenchCAD: A comprehensive, industry-standard benchmark for programmatic CAD","authors":"Zhang et al. and Anthropic","year":"2026","tasks":"1,000-file Vision2Code subset","format":"Voxel IoU with tools","decimals":3,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":0.963},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":0.962},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":0.959},{"slug":"claude-opus-5","name":"Claude Opus 5","score":0.821}]},{"key":"coworkBench","name":"CoWorkBench","fullName":"CoWorkBench","category":"agentic","description":"Qwen's internal benchmark for long-horizon professional work across computer science, finance, law, medicine, and other productivity domains.","paperUrl":"https://qwen.ai/blog?id=qwen3.8","paperTitle":"Qwen3.8-Max: A New Bar for Coding and Cowork","authors":"Qwen Team","year":"2026","tasks":"Long-horizon professional workflows","format":"Provider-run agent score","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":75.3},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":74.8},{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":73.9},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":70.7}]},{"key":"codeforces","name":"Codeforces","fullName":"Codeforces Rating","category":"coding","description":"Competitive-programming rating reported for DeepSeek-V4 thinking-mode evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Competitive programming contests","format":"Rating","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":3471},{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":3206},{"slug":"dots3-note-preview","name":"dots3-note Preview","score":3056},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":3052}]},{"key":"graphwalksBfs256k1m","name":"GraphWalks BFS 256K–1M","fullName":"GraphWalks BFS 256K–1M","category":"reasoning","description":"Breadth-first-search graph traversal on 200 problems with context lengths from 256K to 1M tokens.","paperUrl":"https://deepmind.google/models/evals-methodology/gemini-4-argon","paperTitle":"Gemini 4 Argon evaluation methodology","authors":"Google DeepMind","year":"2026","tasks":"Breadth-first-search graph traversal on 200 problems with context lengths from 256K to 1M tokens.","format":"F1","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":84.2},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":71.8},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":66.8},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":65}]},{"key":"mlsBenchLite","name":"MLS-Bench Lite","fullName":"MLS-Bench Lite","category":"coding","description":"A 30-task subset of MLS-Bench that evaluates whether AI systems can invent generalizable and scalable machine-learning methods.","paperUrl":"https://mls-bench.com/","paperTitle":"MLS-Bench","authors":"MLS-Bench","year":"2026","tasks":"30 machine-learning research tasks","format":"Agentic ML task evaluation","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"kimi-k3","name":"Kimi K3","score":48.3},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":41},{"slug":"step-5-preview","name":"Step 5 Preview","score":40.5},{"slug":"kimi-k2-7-code","name":"Kimi K2.7 Code","score":35.1}]},{"key":"mrcr1m","name":"MRCR 1M","fullName":"MRCR 1M","category":"reasoning","description":"A million-token MRCR long-context retrieval benchmark reported in DeepSeek-V4 model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Million-token retrieval","format":"Long-context retrieval MMR","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":83.5},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":78.7},{"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score":54.1},{"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score":26.6}]},{"key":"odinw13","name":"ODINW13","fullName":"ODINW13","category":"multimodalGrounded","description":"A visual detection and grounding benchmark slice used to compare zero-shot object understanding across diverse domains.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Out-of-distribution object understanding","format":"Detection and grounding","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":51.1},{"slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","score":50.8},{"slug":"qwen3-5-397b","name":"Qwen3.5 397B","score":47},{"slug":"gemini-3-pro","name":"Gemini 3 Pro","score":46.3}]},{"key":"sweAtlasCodebaseQna","name":"SWE-Atlas Codebase QnA","fullName":"SWE-Atlas Codebase QnA","category":"coding","description":"A code-comprehension benchmark covering production repositories across Go, Python, C, and TypeScript.","paperUrl":"https://research.meta.ai/static/muse-spark-1-3-multimodal-evaluation-methodology","paperTitle":"Muse Spark 1.3 Evaluation Methodology","authors":"Meta Superintelligence Labs and Scale AI","year":"2026","tasks":"124 codebase questions across 11 repositories","format":"Mean pass@1","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"ling-3-1-flash","name":"Ling 3.1 Flash","score":64.51},{"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score":59.4},{"slug":"muse-spark-open","name":"Muse Spark Open","score":58},{"slug":"reflection-beam","name":"Beam 1.0 501B-A23B","score":31.8}]},{"key":"toolathlonVerifiedPass3","name":"Toolathlon Verified Pass@3","fullName":"Toolathlon Verified Pass@3","category":"agentic","description":"Fraction of Toolathlon Verified tasks solved in at least one of three trials.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"108 verified real-world tool-use tasks","format":"Pass@3","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":87},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":85.2},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":82.4},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":81.5}]},{"key":"toolathlonVerifiedPass3All","name":"Toolathlon Verified Pass³","fullName":"Toolathlon Verified Pass cubed","category":"agentic","description":"Fraction of Toolathlon Verified tasks solved in all three independent trials.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"108 verified real-world tool-use tasks","format":"All-three-trials pass rate","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":73.1},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":73.1},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":72.2},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":68.5}]},{"key":"toolathlonVerifiedAvgTurns","name":"Toolathlon Verified avg. turns","fullName":"Toolathlon Verified average assistant turns","category":"agentic","description":"Average assistant turns per Toolathlon Verified trajectory.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"108 verified real-world tool-use tasks","format":"Average trajectory length","decimals":1,"modelCount":4,"lowerIsBetter":true,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":23.5},{"slug":"claude-fable-5-1","name":"Claude Fable 5.1","score":23.7},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":26.9},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":31.6}]},{"key":"vision2Web","name":"Vision2Web","fullName":"Vision2Web","category":"multimodalGrounded","description":"A benchmark for converting visual references into functional web implementations.","paperUrl":"https://docs.z.ai/guides/vlm/glm-5v-turbo","paperTitle":"GLM-5V-Turbo","authors":"Z.AI","year":"2026","tasks":"Screenshot-to-web tasks","format":"Visual reference to web implementation","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":69},{"slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","score":64},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":62.9},{"slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","score":62.9}]},{"key":"webArenaVerified","name":"WebArena-Verified","fullName":"WebArena-Verified Browser Agent Benchmark","category":"agentic","description":"WebArena-Verified is an audited release of the WebArena browser-agent benchmark. It rechecks task descriptions, reference answers, and evaluators, and replaces nondeterministic judging with deterministic checks where possible.","paperUrl":"https://openreview.net/forum?id=94tlGxmqkN","paperTitle":"WebArena-Verified: A Fully Audited Benchmark for Web Agents","authors":"Amine El Hattami, Megh Thakkar, Nicolas Chapados, Christopher Pal","year":"2025","tasks":"812 verified tasks; separate 258-task Hard subset","format":"Deterministic end-state task success","decimals":null,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score":69},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":66.8},{"slug":"qwen3-8-27b","name":"Qwen3.8-27B","score":64.8},{"slug":"lfm2-5-24b-a2b","name":"LFM2.5-24B-A2B","score":26.75}]},{"key":"tau2Airline","name":"τ²-bench Airline","fullName":"τ²-Bench Airline Domain","category":"agentic","description":"τ²-bench Airline tests conversational agents on airline customer-service tasks governed by domain policy and database-changing tools.","paperUrl":"https://arxiv.org/abs/2506.07982","paperTitle":"τ²-Bench: Evaluating Conversational Agents in a Dual-Control Environment","authors":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","year":"2025","tasks":"Airline customer-service tasks","format":"Domain success under a published trial policy","decimals":1,"modelCount":4,"lowerIsBetter":false,"top":[{"slug":"trinity-large-thinking","name":"Trinity-Large-Thinking","score":88},{"slug":"nemotron-3-5-super-vl-120b-a12b","name":"Nemotron 3.5 Super VL 120B A12B","score":76.5},{"slug":"ember-1","name":"Ember-1","score":66},{"slug":"zaya1-74b-preview","name":"ZAYA1-74B-Preview","score":56.1}]},{"key":"bankerToolBench","name":"BankerToolBench","fullName":"BankerToolBench","category":"agentic","description":"A display-only provider benchmark for finance-oriented tool-use and agent workflows.","paperUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M3","paperTitle":"MiniMax M3 model card","authors":"MiniMax","year":"2026","tasks":"Finance and banking tool-use tasks","format":"Task success rate","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"hy4-preview","name":"Hy4 preview","score":78.6},{"slug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","score":76.2},{"slug":"minimax-m3","name":"MiniMax M3","score":76.1}]},{"key":"benchCadVision2Code","name":"BenchCAD Vision2Code (no tools)","fullName":"BenchCAD Vision2Code voxel IoU without tools","category":"multimodalGrounded","description":"Generates CadQuery code from multi-view renders and scores geometric similarity by voxel intersection-over-union.","paperUrl":"https://arxiv.org/abs/2605.10865","paperTitle":"BenchCAD: A comprehensive, industry-standard benchmark for programmatic CAD","authors":"Zhang et al. and Anthropic","year":"2026","tasks":"1,000-file Vision2Code subset","format":"Voxel IoU","decimals":3,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":0.747},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":0.73},{"slug":"claude-opus-5","name":"Claude Opus 5","score":0.366}]},{"key":"frontierSwe","name":"FrontierSWE","fullName":"FrontierSWE","category":"coding","description":"An ultra-long-horizon software-engineering benchmark with open-ended implementation, performance, and research tasks designed to challenge frontier coding agents.","paperUrl":"https://www.frontierswe.com/blog","paperTitle":"FrontierSWE: Benchmarking coding agents at the limits of human abilities","authors":"Evan Chu, Rajan Agarwal, Abishek Thangamuthu, Brendan Graham, Justus Mattern","year":"2026","tasks":"17 ultra-long-horizon engineering and research tasks","format":"Mean@5, best@5, average rank, and dominance","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"kimi-k3","name":"Kimi K3","score":81.2},{"slug":"glm-5-3","name":"GLM-5.3","score":78.1},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":73.5}]},{"key":"gmmlu","name":"GMMLU","fullName":"Global MMLU","category":"multilingual","description":"MMLU-style knowledge evaluation across 42 high- and low-resource languages.","paperUrl":"https://arxiv.org/abs/2412.03304","paperTitle":"Global MMLU: Understanding and addressing cultural and linguistic biases in multilingual evaluation","authors":"Singh et al.","year":"2024","tasks":"Knowledge questions across 42 languages","format":"Average accuracy","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":94.3},{"slug":"claude-opus-5","name":"Claude Opus 5","score":92.5},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":92.1}]},{"key":"legalAgentBenchHeldoutAllPass","name":"LAB all-pass (Harvey held-out)","fullName":"Legal Agent Benchmark all-pass rate — Harvey held-out set","category":"agentic","description":"Harvey AI's strict held-out task success rate requiring every rubric criterion to pass.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Harvey AI","year":"2026","tasks":"Harvey-held-out legal-agent tasks","format":"All-criteria pass rate","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":11.7},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":10},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":8.3}]},{"key":"legalAgentBenchHeldoutCriterionPass","name":"LAB criterion-pass (Harvey held-out)","fullName":"Legal Agent Benchmark mean criterion-pass rate — Harvey held-out set","category":"agentic","description":"Harvey AI's mean criterion-level score on its held-out legal-agent evaluation.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Harvey AI","year":"2026","tasks":"Harvey-held-out legal-agent tasks","format":"Mean criterion-pass rate","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":94.1},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":93.1},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":91.2}]},{"key":"milu","name":"MILU","fullName":"Multi-task Indic Language Understanding Benchmark","category":"multilingual","description":"Culturally grounded knowledge comprehension across ten Indic languages and English.","paperUrl":"https://arxiv.org/abs/2411.02538","paperTitle":"MILU: A Multi-task Indic language understanding benchmark","authors":"Verma et al.","year":"2024","tasks":"Knowledge tasks across 11 languages","format":"Average accuracy","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":93.1},{"slug":"claude-opus-5","name":"Claude Opus 5","score":92.1},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":91.6}]},{"key":"officeQa","name":"OfficeQA","fullName":"OfficeQA","category":"multimodalGrounded","description":"Grounded numerical reasoning over a corpus of historical U.S. Treasury Bulletin documents.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Databricks and Anthropic","year":"2026","tasks":"Historical Treasury Bulletin questions","format":"Agentic grounded QA accuracy","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":78.9},{"slug":"claude-opus-5","name":"Claude Opus 5","score":78.1},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":76.9}]},{"key":"perceptionBench","name":"PerceptionBench","fullName":"PerceptionBench (Internal)","category":"multimodalGrounded","description":"Moonshot AI's internal benchmark for atomic visual perception capabilities.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI","year":"2026","tasks":"Internal atomic visual-perception tasks","format":"Internal evaluation score","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":63.5},{"slug":"kimi-k3","name":"Kimi K3","score":58.5},{"slug":"dots3-note-preview","name":"dots3-note Preview","score":53.4}]},{"key":"postTrainBench","name":"PostTrain Bench","fullName":"PostTrain Bench","category":"coding","description":"A software-engineering benchmark for post-training infrastructure and implementation tasks, evaluated through the official Harbor implementation.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI","year":"2026","tasks":"Post-training software-engineering tasks","format":"Harbor agent evaluation","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"glm-5-3","name":"GLM-5.3","score":39.8},{"slug":"kimi-k3","name":"Kimi K3","score":36.6},{"slug":"hy4-preview","name":"Hy4 preview","score":35.6}]},{"key":"proteinDesign","name":"Protein Design","fullName":"Anthropic Protein Design evaluation","category":"knowledge","description":"Generates novel protein sequences under family, topology, globularity, and structural-motif constraints.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"Constrained protein-sequence design tasks","format":"Composite score","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":60.2},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":51},{"slug":"claude-opus-5","name":"Claude Opus 5","score":42.5}]},{"key":"protocolsTroubleshooting","name":"Protocols (troubleshooting)","fullName":"Molecular Biology Protocols Troubleshooting","category":"knowledge","description":"Detects and fixes errors in molecular-biology protocols using document, code, and web-search tools.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"Molecular-biology protocol troubleshooting","format":"Task score","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":73.7},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":67.3},{"slug":"claude-opus-5","name":"Claude Opus 5","score":61.1}]},{"key":"protocolsUnderstanding","name":"Protocols (understanding)","fullName":"Benchling Molecular Biology Protocols Understanding","category":"knowledge","description":"Extends online molecular-biology protocols in additional directions using document and web-search tools.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Benchling and Anthropic","year":"2026","tasks":"Molecular-biology protocol extension","format":"Task score","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":78.4},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":69},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":66.6}]},{"key":"singleCellBench","name":"SingleCellBench","fullName":"LatchBio SingleCellBench","category":"knowledge","description":"Single-cell RNA sequencing analysis tasks spanning common bioinformatics workflows.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"LatchBio and Anthropic","year":"2026","tasks":"195 single-cell RNA sequencing problems","format":"Task score","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":61.2},{"slug":"claude-opus-5","name":"Claude Opus 5","score":60.6},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":59.1}]},{"key":"spatialBenchVerified","name":"SpatialBench Verified","fullName":"LatchBio SpatialBench Verified","category":"knowledge","description":"Analysis of spatial transcriptomics data across externally validated biological problems.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"LatchBio and Anthropic","year":"2026","tasks":"115 externally validated spatial transcriptomics problems","format":"Task score","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":72.5},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":72.5},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":72}]},{"key":"usamo2026","name":"USAMO 2026","fullName":"United States of America Mathematical Olympiad 2026","category":"math","description":"The premier US mathematical olympiad competition, featuring proof-based problems that require deep mathematical insight and rigorous argumentation at the highest competition level.","paperUrl":"https://www.maa.org/math-competitions/usamo","paperTitle":"United States of America Mathematical Olympiad","authors":"Mathematical Association of America","year":"2026","tasks":"6 proof-based problems","format":"Mathematical proof construction","decimals":null,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-mythos-5","name":"Claude Mythos 5","score":97.6},{"slug":"claude-opus-4-8","name":"Claude Opus 4.8","score":96.7},{"slug":"minimax-m3","name":"MiniMax M3","score":85.71}]},{"key":"vulcanCiiV1","name":"VulcanBench CII v1","fullName":"VulcanBench Coding Intelligence Index v1","category":"coding","description":"A post-cutoff software-engineering benchmark with hidden functional tests and regression guards, reported for vendor coding-agent harnesses.","paperUrl":"https://github.com/morganlinton/VulcanBench/blob/main/docs/results/cii-v1-2026-08/README.md","paperTitle":"CII v1 frontier results","authors":"VulcanBench contributors","year":"2026","tasks":"38 validated post-cutoff repository tasks","format":"Pass@1 with vendor coding-agent harnesses","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":96.4},{"slug":"claude-sonnet-5","name":"Claude Sonnet 5","score":89.2},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":86.5}]},{"key":"zeroBenchPython","name":"ZeroBench w/ Python","fullName":"ZeroBench_main with Python","category":"multimodalGrounded","description":"A Python-assisted ZeroBench_main evaluation reported as pass@5.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI / ZeroBench authors","year":"2026","tasks":"Visual reasoning questions with Python","format":"Pass@5","decimals":1,"modelCount":3,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","score":51},{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":49},{"slug":"kimi-k3","name":"Kimi K3","score":41}]},{"key":"arxivMathAugust2026","name":"ArXivMath Aug. 2026 (no tools)","fullName":"ArXivMath August 2026 without tools","category":"math","description":"Final-answer research mathematics problems drawn from recent arXiv papers, including problems based on resolved or refuted conjectures.","paperUrl":"https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf","paperTitle":"Claude Opus 5.5 System Card","authors":"MathArena and Anthropic","year":"2026","tasks":"57 recent research-mathematics problems","format":"Final-answer accuracy","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":91.2},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":86.8}]},{"key":"arxivMathAugust2026WithTools","name":"ArXivMath Aug. 2026 (tools)","fullName":"ArXivMath August 2026 with tools","category":"math","description":"The August 2026 ArXivMath release evaluated with a code-execution sandbox and no internet access.","paperUrl":"https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf","paperTitle":"Claude Opus 5.5 System Card","authors":"MathArena and Anthropic","year":"2026","tasks":"57 recent research-mathematics problems","format":"Final-answer accuracy with tools","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":96.9},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":95.2}]},{"key":"biomedicalImageAnalysis","name":"Biomedical image analysis","fullName":"Anthropic biomedical-image-analysis evaluation","category":"multimodalGrounded","description":"Quantitative biomedical image analysis across segmentation, detection, measurement, classification, registration, reconstruction, and tracking tasks.","paperUrl":"https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf","paperTitle":"Claude Opus 5.5 System Card","authors":"Anthropic","year":"2026","tasks":"91 cases across 31 biomedical imaging tasks","format":"Normalized task metric","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":72.2},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":71.4}]},{"key":"blueprintBench2","name":"Blueprint-Bench 2","fullName":"Blueprint-Bench 2","category":"multimodalGrounded","description":"An agentic spatial reasoning benchmark reported as a normalized score.","paperUrl":"https://x.com/GoogleDeepMind","paperTitle":"Gemini 3.5 Flash launch screenshots","authors":"Google DeepMind","year":"2026","tasks":"Spatial reasoning from blueprints","format":"Normalized score","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-fable","name":"Claude Fable 5","score":38.6},{"slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","score":33.6}]},{"key":"chineseSimpleQa","name":"Chinese-SimpleQA","fullName":"Chinese-SimpleQA","category":"knowledge","description":"A Chinese short-form factuality benchmark reported by DeepSeek for V4 model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Chinese factual questions","format":"Short-form factual QA","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":84.4},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":78.9}]},{"key":"corpusQa1m","name":"CorpusQA 1M","fullName":"CorpusQA 1M","category":"reasoning","description":"A million-token CorpusQA long-context question-answering benchmark reported in DeepSeek-V4 model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Million-token corpus question answering","format":"Long-context QA accuracy","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":62},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":60.5}]},{"key":"cybench","name":"Cybench","fullName":"Cybench","category":"agentic","description":"A cybersecurity benchmark of professional Capture the Flag tasks for measuring autonomous cyber agent capability and risk.","paperUrl":"https://arxiv.org/abs/2408.08926","paperTitle":"Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risks of Language Models","authors":"Andy K. Zhang, Neil Perry, Riya Dulepet, Joey Ji, Celeste Menders, Justin W. Lin, Eliot Jones, Gashon Hussein, Samantha Liu, Donovan Jasper, Pura Peetathawatchai, Ari Glenn, Vikram Sivashankar, Daniel Zamoshchin, Leo Glikbarg, Derek Askaryar, Mike Yang, Teddy Zhang, Rishi Alluri, Nathan Tran, Rinnara Sangpisit, Polycarpos Yiorkadjis, Kenny Osele, Gautham Raghupathi, Dan Boneh, Daniel E. Ho, Percy Liang","year":"2025","tasks":"40 professional CTF tasks","format":"Cybersecurity agent task completion","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"mistral-large-4","name":"Mistral Large 4","score":93},{"slug":"muse-spark-1-1","name":"Muse Spark 1.1","score":92.9}]},{"key":"dsBenchFullStack","name":"DSBench-FullStack","fullName":"DeepSeek DSBench FullStack","category":"coding","description":"DeepSeek's internal full-stack coding-agent benchmark.","paperUrl":"https://api-docs.deepseek.com/zh-cn/updates/","paperTitle":"DeepSeek V4 Flash 0731 update","authors":"DeepSeek-AI","year":"2026","tasks":"Internal full-stack coding-agent tasks","format":"Provider-reported score","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":71.1},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":68.7}]},{"key":"dsBenchHard","name":"DSBench-Hard","fullName":"DeepSeek DSBench Hard","category":"coding","description":"DeepSeek's internal hard coding-agent benchmark.","paperUrl":"https://api-docs.deepseek.com/zh-cn/updates/","paperTitle":"DeepSeek V4 Flash 0731 update","authors":"DeepSeek-AI","year":"2026","tasks":"Internal hard coding-agent tasks","format":"Provider-reported score","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":67.2},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":59.6}]},{"key":"proteinBinderDesign","name":"De novo protein-binder design","fullName":"Anthropic de novo protein-binder design evaluation","category":"knowledge","description":"Designs diverse miniprotein binders against target protein structures in a GPU-equipped research sandbox.","paperUrl":"https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf","paperTitle":"Claude Opus 5.5 System Card","authors":"Anthropic","year":"2026","tasks":"15 protein targets","format":"Composite structural design score","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":82.6},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":82.3}]},{"key":"frontierScienceResearch","name":"FrontierScience Research","fullName":"FrontierScience Research","category":"knowledge","description":"A research-focused FrontierScience evaluation variant for scientific investigation and problem solving.","paperUrl":"https://ai.meta.com/static-resource/muse-spark-eval-methodology","paperTitle":"Muse Spark Eval Methodology","authors":"Meta AI","year":"2026","tasks":"Scientific research problems","format":"Research evaluation","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"apodex-1-1","name":"Apodex 1.1","score":63.3},{"slug":"gpt-5-4-pro","name":"GPT-5.4 Pro","score":36.7}]},{"key":"gdpPdf","name":"GDP.pdf (no tools)","fullName":"GDP.pdf mean criteria pass rate without tools","category":"multimodalGrounded","description":"Professional document understanding over 100 real-world PDFs from ten domains.","paperUrl":"https://surgehq.ai/blog/gdp-pdf-can-100b-ai-models-master-the-documents-that-run-the-world","paperTitle":"GDP.pdf","authors":"Surge AI and Anthropic","year":"2026","tasks":"100 professional document prompts","format":"Mean criteria pass rate","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":83.4},{"slug":"dots3-note-preview","name":"dots3-note Preview","score":60.7}]},{"key":"geneBenchPro","name":"GeneBench-Pro","fullName":"GeneBench-Pro","category":"reasoning","description":"A multistage statistical-reasoning benchmark for genomics and biological-data analysis agents.","paperUrl":"https://cdn.openai.com/pdf/21938268-21af-442f-af93-3b2249afb241/genebench-pro.pdf","paperTitle":"GeneBench-Pro: Evaluating Multistage Statistical Reasoning","authors":"OpenAI","year":2026,"tasks":"129 genomics statistical-analysis workflows","format":"Eval-level pass rate across dependent analysis decisions","decimals":null,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":37.8},{"slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","score":28.7}]},{"key":"graphwalksBfs128k","name":"Graphwalks BFS 128K","fullName":"Graphwalks BFS 0K-128K","category":"reasoning","description":"Long-context graph traversal benchmark using breadth-first search tasks.","paperUrl":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","paperTitle":"Introducing GPT-5.4 mini and nano","authors":"OpenAI","year":"2026","tasks":"Graph traversal tasks","format":"Long-context graph reasoning","decimals":null,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":99.7},{"slug":"mai-thinking-1","name":"MAI-Thinking-1","score":90}]},{"key":"imo2026","name":"IMO 2026","fullName":"International Mathematical Olympiad 2026","category":"math","description":"Proof-based olympiad performance on all six IMO 2026 problems.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"6 proof-based problems","format":"Official-style proof score","decimals":0,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":42},{"slug":"apodex-1-1","name":"Apodex 1.1","score":30.5}]},{"key":"kimiCodeBenchV2","name":"Kimi Code Bench v2","fullName":"Kimi Code Bench v2","category":"coding","description":"A Moonshot AI internal coding-agent benchmark for realistic software-engineering tasks across mainstream programming languages and production technology stacks.","paperUrl":"https://huggingface.co/moonshotai/Kimi-K2.7-Code","paperTitle":"Kimi K2.7 Code","authors":"Moonshot AI","year":"2026","tasks":"Realistic coding-agent tasks","format":"Coding-agent pass rate","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"kimi-k3","name":"Kimi K3","score":72.9},{"slug":"kimi-k2-7-code","name":"Kimi K2.7 Code","score":62}]},{"key":"liveCodeBenchPass1Cot","name":"LiveCodeBench Pass@1-COT","fullName":"LiveCodeBench Pass@1 with Chain-of-Thought","category":"coding","description":"This lane contains DeepSeek's LiveCodeBench Pass@1-COT results. The explicit metric and prompting label keeps them separate from generic and version-specific LiveCodeBench rows.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/DeepSeek_V4.pdf?download=true","paperTitle":"DeepSeek-V4 technical report","authors":"DeepSeek","year":"2026","tasks":"DeepSeek-V4 report evaluation window","format":"Pass@1-COT competitive programming results","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","score":93.5},{"slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","score":91.6}]},{"key":"mcpAtlasClaimCoverage","name":"MCP-Atlas claim coverage","fullName":"MCP-Atlas mean claim coverage","category":"agentic","description":"Average coverage of required claims in answers produced during real-world MCP tool-use workflows.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"Production-like multi-server MCP workflows","format":"Mean claim coverage","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":89.1},{"slug":"pokee-isaac-28b","name":"Pokee-Isaac 28B","score":74.59}]},{"key":"mleBenchLite","name":"MLE-Bench Lite","fullName":"MLE-Bench Lite","category":"agentic","description":"A lightweight machine-learning competition benchmark that measures whether models can iteratively train, evaluate, and improve ML systems in low-resource settings.","paperUrl":"https://www.minimax.io/news/minimax-m27-en","paperTitle":"MiniMax M2.7: Early Echoes of Self-Evolution","authors":"MiniMax","year":"2026","tasks":"Low-resource ML competitions","format":"Autonomous iterative ML optimization","decimals":null,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"atria-dawn-preview","name":"Atria Dawn Preview","score":86.2},{"slug":"minimax-m2-7","name":"MiniMax M2.7","score":66.6}]},{"key":"mmClawBench","name":"MM-ClawBench","fullName":"MM-ClawBench","category":"agentic","description":"An OpenClaw-derived agent benchmark covering practical work and life tasks such as office document delivery, research, planning, and code maintenance.","paperUrl":"https://www.minimax.io/news/minimax-m27-en","paperTitle":"MiniMax M2.7: Early Echoes of Self-Evolution","authors":"MiniMax","year":"2026","tasks":"OpenClaw-style real-world tasks","format":"Agent workflow evaluation","decimals":null,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"minimax-m2-7","name":"MiniMax M2.7","score":62.7},{"slug":"mimo-v2-5","name":"MiMo-V2.5","score":23.8}]},{"key":"mrcrv2_128_256","name":"MRCR v2 128K-256K","fullName":"OpenAI MRCR v2 8-needle 128K-256K","category":"reasoning","description":"MRCR v2 slice focused on very long contexts at 128K-256K lengths.","paperUrl":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","paperTitle":"Introducing GPT-5.4 mini and nano","authors":"OpenAI","year":"2026","tasks":"8-needle retrieval tasks","format":"Very-long-context retrieval","decimals":null,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"gpt-5-5","name":"GPT-5.5","score":87.5},{"slug":"claude-opus-4-7-adaptive","name":"Claude Opus 4.7 (Adaptive)","score":59.2}]},{"key":"mrcrv2_256_512","name":"MRCR v2 256K-512K","fullName":"OpenAI MRCR v2 8-needle 256K-512K","category":"reasoning","description":"MRCR v2 slice focused on retrieval across 256K-512K-token contexts.","paperUrl":"https://research.meta.ai/static/muse-spark-1-3-multimodal-evaluation-methodology","paperTitle":"Muse Spark 1.3 Evaluation Methodology","authors":"Meta Superintelligence Labs","year":"2026","tasks":"100 eight-needle retrieval examples","format":"Mean sequence-matcher ratio","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":100},{"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score":98.5}]},{"key":"mrcrv2_512_1m","name":"MRCR v2 512K-1M","fullName":"OpenAI MRCR v2 8-needle 512K-1M","category":"reasoning","description":"MRCR v2 slice focused on retrieval across 512K-1M-token contexts.","paperUrl":"https://research.meta.ai/static/muse-spark-1-3-multimodal-evaluation-methodology","paperTitle":"Muse Spark 1.3 Evaluation Methodology","authors":"Meta Superintelligence Labs","year":"2026","tasks":"100 eight-needle retrieval examples","format":"Mean sequence-matcher ratio","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"muse-spark-1-3","name":"Muse Spark 1.3","score":98.1},{"slug":"gpt-6-astra","name":"GPT-6 Astra","score":96.3}]},{"key":"mrcrv2_64_128","name":"MRCR v2 64K-128K","fullName":"OpenAI MRCR v2 8-needle 64K-128K","category":"reasoning","description":"MRCR v2 slice focused on long-context retrieval at 64K-128K lengths.","paperUrl":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","paperTitle":"Introducing GPT-5.4 mini and nano","authors":"OpenAI","year":"2026","tasks":"8-needle retrieval tasks","format":"Long-context retrieval","decimals":null,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","score":97},{"slug":"gpt-5-5","name":"GPT-5.5","score":83.1}]},{"key":"medicinalChemistry","name":"Medicinal chemistry","fullName":"Anthropic medicinal-chemistry evaluation","category":"knowledge","description":"Selects compounds with the highest or lowest measured ADME-related property from molecular structures and reference values.","paperUrl":"https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf","paperTitle":"Claude Opus 5.5 System Card","authors":"Anthropic","year":"2026","tasks":"504 molecular-property questions","format":"Accuracy","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":65.3},{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":63.5}]},{"key":"morphologyToMolecule","name":"Morphology-to-molecule matching","fullName":"Axiom Bio morphology-to-molecule matching","category":"knowledge","description":"Matches multi-channel cellular imaging data to the molecule-dose perturbations that produced it.","paperUrl":"https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf","paperTitle":"Claude Opus 5.5 System Card","authors":"Axiom Bio and Anthropic","year":"2026","tasks":"Four multi-channel cell-imaging perturbation-matching tasks","format":"Matching accuracy","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":34},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":25}]},{"key":"musr","name":"MuSR","fullName":"Testing the Limits of Chain-of-thought with Multistep Soft Reasoning","category":"reasoning","description":"A dataset for evaluating language models on multistep soft reasoning tasks specified in natural language narratives. Tests the ability to perform complex, structured reasoning.","paperUrl":"https://arxiv.org/abs/2310.16049","paperTitle":"MuSR: Testing the Limits of Chain-of-thought with Multistep Soft Reasoning","authors":"Zayne Sprague, Xi Ye, Kaj Bostrom, Swarat Chaudhuri, Greg Durrett","year":"2023","tasks":"Multi-step reasoning","format":"Narrative-based reasoning","decimals":null,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"ternary-bonsai-8b","name":"Ternary Bonsai 8B","score":56.2},{"slug":"bonsai-8b","name":"1-bit Bonsai 8B","score":50}]},{"key":"osWorld","name":"OSWorld","fullName":"OSWorld","category":"agentic","description":"A computer-use benchmark for GUI task completion across the broader OSWorld task suite.","paperUrl":"https://docs.z.ai/guides/vlm/glm-5v-turbo","paperTitle":"GLM-5V-Turbo","authors":"Z.AI","year":"2026","tasks":"Computer-use tasks","format":"Interactive GUI evaluation","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-opus-4-5","name":"Claude Opus 4.5","score":66.3},{"slug":"nemotron-3-nano-omni-30b-a3b","name":"Nemotron 3 Nano Omni 30B A3B","score":47.4}]},{"key":"proteinDesignLibraryRanking","name":"Protein Design library ranking","fullName":"Anthropic protein-design library-ranking task","category":"knowledge","description":"Ranks held-out protein designs for experimental testing using wet-lab campaign measurements and a selection objective.","paperUrl":"https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf","paperTitle":"Claude Opus 5.5 System Card","authors":"Anthropic","year":"2026","tasks":"83 protein-design library-ranking problems","format":"Chance-normalized average precision","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5-5","name":"Claude Opus 5.5","score":56},{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":54.8}]},{"key":"videoMme","name":"Video-MME","fullName":"Video-MME","category":"multimodalGrounded","description":"A comprehensive benchmark for multimodal large language models on video understanding, covering temporal reasoning, perception, and question answering over videos.","paperUrl":"https://mme-benchmark.github.io/","paperTitle":"Video-MME benchmark","authors":"Video-MME benchmark team","year":"2024","tasks":"Video understanding","format":"Video QA and analysis","decimals":1,"modelCount":2,"lowerIsBetter":false,"top":[{"slug":"seed-2-1-pro","name":"Seed 2.1 Pro","score":89.2},{"slug":"kimi-k2-5","name":"Kimi K2.5","score":87.4}]},{"key":"aokvqa","name":"A-OKVQA","fullName":"A Benchmark for Visual Question Answering using World Knowledge","category":"multimodalGrounded","description":"A visual question answering benchmark whose questions cannot be answered from the image alone and require commonsense or world knowledge.","paperUrl":"https://arxiv.org/abs/2206.01718","paperTitle":"A-OKVQA: A Benchmark for Visual Question Answering using World Knowledge","authors":"Dustin Schwenk, Apoorv Khandelwal, Christopher Clark, Kenneth Marino, Roozbeh Mottaghi","year":"2022","tasks":"Knowledge-grounded visual question answering","format":"Multiple choice and direct answer","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"ternary-bonsai-2-27b","name":"Ternary Bonsai 2 27B","score":86.81}]},{"key":"agieval","name":"AGIEval","fullName":"AGIEval","category":"knowledge","description":"A human-centric exam benchmark for general knowledge and reasoning reported in DeepSeek-V4 base-model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"General academic and professional exam questions","format":"Exact match","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"soofi-s-30b-a3b","name":"Soofi S 30B-A3B","score":66.9}]},{"key":"arxivMathJune2026","name":"ArXivMath Jun. 2026 (no tools)","fullName":"ArXivMath June 2026 without tools","category":"math","description":"Final-answer research mathematics problems drawn from recent arXiv abstracts.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"MathArena and Anthropic","year":"2026","tasks":"49 recent research-mathematics problems","format":"Final-answer accuracy","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":90.8}]},{"key":"arxivMathJune2026WithTools","name":"ArXivMath Jun. 2026 (tools)","fullName":"ArXivMath June 2026 with tools","category":"math","description":"Final-answer research mathematics problems drawn from recent arXiv abstracts with tool access.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"MathArena and Anthropic","year":"2026","tasks":"49 recent research-mathematics problems","format":"Final-answer accuracy with tools","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":91.3}]},{"key":"automationBenchZapier106","name":"AutomationBench (Zapier 1.0.6)","fullName":"AutomationBench, Zapier private evaluation set release 1.0.6","category":"agentic","description":"An agent benchmark for completing business workflows across simulated applications and API endpoints.","paperUrl":"https://arxiv.org/abs/2604.18934","paperTitle":"AutomationBench","authors":"Daniel Shepard and Robin Salimans","year":"2026","tasks":"600+ held-out workflows across 47 simulated applications and six business domains","format":"Task pass rate","decimals":2,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","score":44.7}]},{"key":"bigCodeBench","name":"BigCodeBench","fullName":"BigCodeBench","category":"coding","description":"A code-generation benchmark reported in DeepSeek-V4 base-model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Code generation tasks","format":"Pass@1","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"ternary-bonsai-2-27b","name":"Ternary Bonsai 2 27B","score":58.07}]},{"key":"multiAgentBrowseCompPrerelease","name":"BrowseComp (10-agent, prerelease)","fullName":"Multi-Agent BrowseComp — 10-agent team prerelease configuration","category":"agentic","description":"BrowseComp accuracy from ten collaborating Opus 5 agents on a pre-release model and unreleased effort configuration.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"BrowseComp web-research tasks","format":"10-agent team accuracy","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":93.6}]},{"key":"browseCompPlus","name":"BrowseComp+","fullName":"BrowseComp-Plus","category":"agentic","description":"Evaluates research agents against a fixed document corpus, separately from live-web BrowseComp.","paperUrl":"https://arxiv.org/abs/2508.06600","paperTitle":"BrowseComp-Plus: A More Fair and Transparent Evaluation Benchmark of Deep-Research Agent","authors":"Zijian Chen et al.","year":"2025","tasks":null,"format":null,"decimals":2,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"mercury-3","name":"Mercury 3","score":55.18}]},{"key":"cmmlu","name":"CMMLU","fullName":"Chinese Massive Multitask Language Understanding","category":"knowledge","description":"A Chinese multitask academic benchmark reported in DeepSeek-V4 base-model evaluations.","paperUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf","paperTitle":"DeepSeek-V4 Technical Report","authors":"DeepSeek-AI","year":"2026","tasks":"Chinese academic QA","format":"Exact match","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"longcat-flash-lite-sparse","name":"LongCat-Flash-Lite-Sparse","score":84.25}]},{"key":"charxivOverall","name":"CharXiv (overall)","fullName":"CharXiv Descriptive and Reasoning Combined","category":"multimodalGrounded","description":"A scientific chart understanding benchmark scored across both the descriptive and reasoning splits rather than the reasoning split alone.","paperUrl":"https://charxiv.github.io/","paperTitle":"CharXiv: Charting Gaps in Realistic Chart Understanding in Multimodal LLMs","authors":"CharXiv authors","year":"2024","tasks":"Scientific chart description and reasoning","format":"Chart understanding and reasoning","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"ternary-bonsai-2-27b","name":"Ternary Bonsai 2 27B","score":80}]},{"key":"deckBench","name":"DECK-Bench","fullName":"DECK-Bench (Internal)","category":"agentic","description":"Moonshot AI's internal benchmark for presentation and deck-production workflows.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI","year":"2026","tasks":"Internal presentation workflows","format":"Internal evaluation score","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"kimi-k3","name":"Kimi K3","score":73.5}]},{"key":"designArenaAgenticWebDev","name":"Design Arena Agentic Web Dev","fullName":"Design Arena Agentic Web Dev Elo","category":"agentic","description":"A display-only Elo rating from blinded comparisons of multi-file web applications built by coding agents.","paperUrl":"https://intelligence.ai/leaderboard/webapps","paperTitle":"Design Arena Web Dev (Agentic) Leaderboard","authors":"Design Arena / Intelligence","year":"2026","tasks":"Multi-file web application development","format":"Elo from blinded human preferences","decimals":0,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"inkling","name":"Inkling","score":1257}]},{"key":"eeBench","name":"EEBench","fullName":"EEBench V1 core corpus","category":"coding","description":"An electrical-engineering agent benchmark maintained by atopile.","paperUrl":"https://www.eebench.org/","paperTitle":"EEBench leaderboard","authors":"atopile","year":"2026","tasks":"Electrical-engineering design tasks","format":"Agent task score","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"grok-4-7","name":"Grok 4.7","score":64}]},{"key":"frontierScience","name":"FrontierScience","fullName":"FrontierScience","category":"knowledge","description":"A benchmark for research-level scientific reasoning, designed to separate frontier models on difficult science tasks that mix domain knowledge with deep reasoning.","paperUrl":"https://openai.com/index/frontierscience/","paperTitle":"FrontierScience","authors":"OpenAI","year":"2026","tasks":"Research-level science tasks","format":"Scientific reasoning benchmark","decimals":null,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"gpt-5-4-pro","name":"GPT-5.4 Pro","score":36.7}]},{"key":"gdpPdfWithTools","name":"GDP.pdf (tools)","fullName":"GDP.pdf mean criteria pass rate with tools","category":"multimodalGrounded","description":"Professional document understanding with a container, standard libraries, and image cropping.","paperUrl":"https://surgehq.ai/blog/gdp-pdf-can-100b-ai-models-master-the-documents-that-run-the-world","paperTitle":"GDP.pdf","authors":"Surge AI and Anthropic","year":"2026","tasks":"100 professional document prompts","format":"Mean criteria pass rate with tools","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":85.5}]},{"key":"gdpvalRubrics","name":"GDPval rubrics","fullName":"GDPval rubrics","category":"agentic","description":"A display-only provider-table GDPval rubric score for economically valuable work tasks.","paperUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M3","paperTitle":"MiniMax M3 model card","authors":"MiniMax","year":"2026","tasks":"Economically valuable work tasks","format":"Rubric score","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"minimax-m3","name":"MiniMax M3","score":74.7}]},{"key":"ipho2025Theory","name":"IPhO 2025 (Theory)","fullName":"International Physics Olympiad 2025 (Theory)","category":"math","description":"The three official theory problems from the 2025 International Physics Olympiad, scored with blinded human evaluation.","paperUrl":"https://ai.meta.com/static-resource/muse-spark-eval-methodology","paperTitle":"Muse Spark Eval Methodology","authors":"Meta AI","year":"2026","tasks":"3 olympiad theory problems","format":"Physics olympiad theory","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"gpt-5-4-pro","name":"GPT-5.4 Pro","score":93.5}]},{"key":"kernelBenchHard","name":"KernelBench Hard","fullName":"KernelBench Hard","category":"coding","description":"A display-only benchmark for difficult GPU kernel implementation and optimization tasks.","paperUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M3","paperTitle":"MiniMax M3 model card","authors":"MiniMax","year":"2026","tasks":"Hard GPU kernel coding tasks","format":"Task success rate","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"minimax-m3","name":"MiniMax M3","score":28.8}]},{"key":"kimiClaw247","name":"Kimi Claw 24/7","fullName":"Kimi Claw 24/7 Bench","category":"agentic","description":"A Moonshot AI internal long-horizon agent benchmark for persistent professional coworking tasks.","paperUrl":"https://huggingface.co/moonshotai/Kimi-K2.7-Code","paperTitle":"Kimi K2.7 Code","authors":"Moonshot AI","year":"2026","tasks":"17 professional scenarios, 610 evaluation points","format":"Average pass rate across repeated OpenClaw runs","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"kimi-k2-7-code","name":"Kimi K2.7 Code","score":46.9}]},{"key":"legalAgentBenchAllPass","name":"LAB all-pass (Anthropic harness)","fullName":"Legal Agent Benchmark all-pass rate — Anthropic harness","category":"agentic","description":"Strict task success requiring every expert-written legal-work rubric criterion to pass.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Harvey AI and Anthropic","year":"2026","tasks":"1,235 legal-agent tasks","format":"All-criteria pass rate","decimals":2,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":23.58}]},{"key":"legalAgentBenchCriterionPass","name":"LAB criterion-pass (Anthropic harness)","fullName":"Legal Agent Benchmark mean criterion-pass rate — Anthropic harness","category":"agentic","description":"Mean fraction of expert-written rubric criteria passed across legal-agent tasks.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Harvey AI and Anthropic","year":"2026","tasks":"1,235 legal-agent tasks","format":"Mean criterion-pass rate","decimals":2,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":93.74}]},{"key":"locaBench256k","name":"LOCA-Bench 256K","fullName":"LOCA-Bench 256K","category":"agentic","description":"Tests long-context agents as tool environments grow to a 256K-token environment description.","paperUrl":"https://arxiv.org/abs/2602.07962","paperTitle":"LOCA-bench: Benchmarking Language Agents Under Controllable Environment Growth","authors":"Weihao Zeng, Yuzhen Huang, Junxian He","year":"2026","tasks":null,"format":"Agent tasks with controlled tool-environment growth","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","score":60.3}]},{"key":"liquidExtractSchemaF1","name":"Liquid Extract F1","fullName":"Liquid image-to-JSON extraction schema consistency F1","category":"multimodalGrounded","description":"A display-only Liquid AI extraction metric measuring field-name agreement between requested schema fields and extracted JSON fields.","paperUrl":"https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B-Extract","paperTitle":"LiquidAI LFM2.5-VL Extract model cards","authors":"Liquid AI","year":"2026","tasks":"Image-to-JSON extraction","format":"Schema field F1","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"lfm2-5-vl-1-6b-extract","name":"LFM2.5-VL-1.6B-Extract","score":99.6}]},{"key":"liquidExtractJsonValidity","name":"Liquid Extract JSON Validity","fullName":"Liquid image-to-JSON extraction JSON validity","category":"multimodalGrounded","description":"A display-only Liquid AI extraction metric measuring the share of image-to-JSON outputs that parse as strict JSON.","paperUrl":"https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B-Extract","paperTitle":"LiquidAI LFM2.5-VL Extract model cards","authors":"Liquid AI","year":"2026","tasks":"Image-to-JSON extraction","format":"Strict JSON parseability rate","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"lfm2-5-vl-1-6b-extract","name":"LFM2.5-VL-1.6B-Extract","score":99.6}]},{"key":"liquidExtractVlmJudge","name":"Liquid Extract VLM Judge","fullName":"Liquid image-to-JSON extraction VLM judge score","category":"multimodalGrounded","description":"A display-only Liquid AI extraction metric measuring judged agreement between extracted values and the source image.","paperUrl":"https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B-Extract","paperTitle":"LiquidAI LFM2.5-VL Extract model cards","authors":"Liquid AI","year":"2026","tasks":"Image-to-JSON extraction","format":"VLM-judged extraction accuracy","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"lfm2-5-vl-1-6b-extract","name":"LFM2.5-VL-1.6B-Extract","score":90.6}]},{"key":"longContextTranslation","name":"Long-context translation","fullName":"Long-context translation evaluation (Cohere)","category":"multilingual","description":"Cohere's long-document translation evaluation: two book chapters translated in a single call, scored per paragraph with xCOMET-XL.","paperUrl":"https://cohere.com/blog/north-small-translate","paperTitle":"Introducing North Small Translate","authors":"Cohere","year":"2026","tasks":"Two book chapters per call","format":"xCOMET-XL quality score","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"north-small-translate-1-0","name":"North Small Translate","score":48.9}]},{"key":"mcpMarkVerified","name":"MCP Mark Verified","fullName":"MCPMark-Verified","category":"agentic","description":"A human-verified edition of MCPMark for MCP tool use across Notion, GitHub, Filesystem, Postgres, and Playwright server environments.","paperUrl":"https://mcpmark.ai/","paperTitle":"MCPMark","authors":"MCPMark","year":"2026","tasks":"MCP tool-use tasks across five server environments","format":"Interactive MCP task completion","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"kimi-k2-7-code","name":"Kimi K2.7 Code","score":81.1}]},{"key":"mkqa11","name":"MKQA-11","fullName":"MKQA-11 multilingual retrieval","category":"multilingual","description":"A display-only multilingual QA retrieval benchmark reported by Liquid AI for LFM2.5 retriever models, using Recall@20 across 11 languages.","paperUrl":"https://www.liquid.ai/blog/lfm2-5-retrievers","paperTitle":"LFM2.5 Retrievers: Bi-directional LFMs for Fast Multilingual Search","authors":"Liquid AI","year":"2026","tasks":"Cross-lingual open-domain QA retrieval","format":"Recall@20 average","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"lfm2-5-embedding-350m","name":"LFM2.5-Embedding-350M","score":69.1}]},{"key":"mmSearchPlus","name":"MMSearch-Plus","fullName":"MMSearch-Plus","category":"multimodalGrounded","description":"A harder MMSearch variant for multimodal retrieval and grounded tool-use workflows.","paperUrl":"https://docs.z.ai/guides/vlm/glm-5v-turbo","paperTitle":"GLM-5V-Turbo","authors":"Z.AI","year":"2026","tasks":"Hard multimodal search tasks","format":"Advanced mixed-media retrieval benchmark","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"qwen3-7-plus","name":"Qwen3.7 Plus","score":41.4}]},{"key":"mStar","name":"MStar","fullName":"MStar","category":"multimodalGrounded","description":"A general visual question-answering benchmark used in provider tables for real-image reasoning quality.","paperUrl":"https://qwen.ai/blog?id=qwen3.6","paperTitle":"Qwen3.6 launch benchmarks","authors":"Qwen","year":"2026","tasks":"Real-image visual QA","format":"Image-grounded QA","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"qwen3-6-27b","name":"Qwen3.6-27B","score":81.4}]},{"key":"mobileWorld","name":"MobileWorld","fullName":"MobileWorld","category":"agentic","description":"A mobile-use agent benchmark for completing interactive tasks in smartphone environments.","paperUrl":"https://qwen.ai/blog?id=qwen3.8","paperTitle":"Qwen3.8-Max: A New Bar for Coding and Cowork","authors":"Qwen Team","year":"2026","tasks":"Interactive mobile-device workflows","format":"Mobile agent task score","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":77.8}]},{"key":"multiSweBench","name":"Multi-SWE Bench","fullName":"Multi-SWE Bench","category":"coding","description":"A multi-language software-engineering benchmark that measures repository-level bug fixing and implementation across more than one programming ecosystem.","paperUrl":"https://www.minimax.io/news/minimax-m27-en","paperTitle":"MiniMax M2.7: Early Echoes of Self-Evolution","authors":"MiniMax","year":"2026","tasks":"Multi-language repo tasks","format":"Repository task completion","decimals":null,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"minimax-m2-7","name":"MiniMax M2.7","score":52.7}]},{"key":"multiChallenge","name":"MultiChallenge (provider reports)","fullName":"MultiChallenge: provider-reported runs","category":"instructionFollowing","description":"Tests instruction retention, memory of user information, versioned editing, and coherence over multi-turn conversations.","paperUrl":"https://labs.scale.com/leaderboard/multichallenge","paperTitle":"MultiChallenge","authors":"Scale AI","year":"2025","tasks":null,"format":null,"decimals":2,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"mercury-3","name":"Mercury 3","score":60.35}]},{"key":"nanoBeirMultilingual","name":"NanoBEIR Multilingual","fullName":"NanoBEIR Multilingual Extended","category":"multilingual","description":"A display-only multilingual retrieval benchmark reported by Liquid AI for LFM2.5 retriever models, using NDCG@10 across 11 languages.","paperUrl":"https://www.liquid.ai/blog/lfm2-5-retrievers","paperTitle":"LFM2.5 Retrievers: Bi-directional LFMs for Fast Multilingual Search","authors":"Liquid AI","year":"2026","tasks":"Multilingual document retrieval","format":"NDCG@10 average","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"lfm2-5-embedding-350m","name":"LFM2.5-Embedding-350M","score":57.7}]},{"key":"omniDocBench","name":"OmniDocBench","fullName":"OmniDocBench","category":"multimodalGrounded","description":"A document-understanding benchmark for parsing and reasoning over complex document layouts.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI / OmniDocBench authors","year":"2026","tasks":"Complex document-understanding tasks","format":"Document-understanding score","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"kimi-k3","name":"Kimi K3","score":91.1}]},{"key":"omniDocBench16","name":"OmniDocBench 1.6","fullName":"OmniDocBench v1.6","category":"multimodalGrounded","description":"A document parsing benchmark whose overall score composites edit distance, CDM and TEDS sub-scores across diverse PDF layouts.","paperUrl":"https://arxiv.org/abs/2412.07626","paperTitle":"OmniDocBench: Benchmarking Diverse PDF Document Parsing with Comprehensive Annotations","authors":"Linke Ouyang et al.","year":"2024","tasks":"Document parsing and extraction","format":"Composite of edit distance, CDM and TEDS","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"ternary-bonsai-2-27b","name":"Ternary Bonsai 2 27B","score":89.13}]},{"key":"organicChemistryV2","name":"Organic chemistry V2","fullName":"Anthropic Organic Chemistry V2 evaluation","category":"knowledge","description":"Chemistry tasks covering spectroscopy, synthesis planning, reaction prediction, and chemical structure images.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"Organic chemistry reasoning tasks","format":"Task score","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":61.6}]},{"key":"paperBench","name":"PaperBench","fullName":"PaperBench","category":"coding","description":"A research-reproduction benchmark that asks agents to recreate the contributions of AI papers from the paper alone.","paperUrl":"https://qwen.ai/blog?id=qwen3.8","paperTitle":"Qwen3.8-Max: A New Bar for Coding and Cowork","authors":"Qwen Team","year":"2026","tasks":"AI research-paper reproduction","format":"Long-horizon agent evaluation","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":93}]},{"key":"programBenchEpisode1","name":"ProgramBench (episode 1)","fullName":"ProgramBench hidden-test pass rate after episode 1","category":"coding","description":"Program-reconstruction hidden-test pass rate after the first of five sequential long-context episodes.","paperUrl":"https://arxiv.org/abs/2605.03546","paperTitle":"ProgramBench: Can language models rebuild programs from scratch?","authors":"Yang et al.","year":"2026","tasks":"166 golden program-reconstruction tasks","format":"Hidden-test pass rate after episode 1","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":83}]},{"key":"proteinGymHard","name":"ProteinGym Hard","fullName":"ProteinGym Hard","category":"knowledge","description":"Predicts mutation effects by ranking mutant protein sequences against wild type and comparing against laboratory measurements.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Anthropic","year":"2026","tasks":"Hard protein mutation-effect ranking tasks","format":"Rank correlation","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":47.7}]},{"key":"qwenReactBench","name":"QwenReactBench","fullName":"QwenReactBench","category":"coding","description":"Qwen's internal benchmark for building and rendering bilingual React projects across seven categories.","paperUrl":"https://qwen.ai/blog?id=qwen3.8","paperTitle":"Qwen3.8-Max: A New Bar for Coding and Cowork","authors":"Qwen Team","year":"2026","tasks":"Bilingual React project construction","format":"Bradley-Terry/Elo rating","decimals":0,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"qwen3-8-max","name":"Qwen3.8 Max","score":1724}]},{"key":"riemannBench","name":"RiemannBench (no tools)","fullName":"RiemannBench without tools","category":"math","description":"Research-level mathematics problems with unique programmatically verified closed-form answers.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Surge AI and Anthropic","year":"2026","tasks":"25 private research-level mathematics problems","format":"Accuracy without tools","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":60}]},{"key":"riemannBenchWithTools","name":"RiemannBench (tools)","fullName":"RiemannBench with tools","category":"math","description":"Research-level mathematics problems with unique programmatically verified closed-form answers and tool access.","paperUrl":"https://www-cdn.anthropic.com/c5fbac3f0b1280a933ebd26d3cb8bb9f5bdeaf48/Claude%20Opus%205%20System%20Card.pdf","paperTitle":"Claude Opus 5 System Card","authors":"Surge AI and Anthropic","year":"2026","tasks":"25 private research-level mathematics problems","format":"Accuracy with tools","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"claude-opus-5","name":"Claude Opus 5","score":79}]},{"key":"sobValueAcc","name":"SOB Value Acc","fullName":"Structured Output Benchmark Value Accuracy","category":"instructionFollowing","description":"A structured-output benchmark from Interfaze measuring whether extracted JSON leaf values exactly match verified ground truth.","paperUrl":"https://interfaze.ai/leaderboards/structured-output-benchmark","paperTitle":"Structured Output Benchmark Leaderboard","authors":"Interfaze","year":"2026","tasks":"Structured output extraction","format":"Value accuracy","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"interfaze-beta","name":"Interfaze Beta","score":79.5}]},{"key":"svgBench","name":"SVG-Bench","fullName":"SVG-Bench","category":"coding","description":"A display-only provider benchmark for generating or manipulating SVG outputs from natural-language requirements.","paperUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M3","paperTitle":"MiniMax M3 model card","authors":"MiniMax","year":"2026","tasks":"SVG generation and editing tasks","format":"Task success rate","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"minimax-m3","name":"MiniMax M3","score":63.7}]},{"key":"sweRefactorBench","name":"SWE Refactor Bench","fullName":"SWE Refactor Bench: Can Coding Agents Complete a Long-Horizon, Whole-Repository Stack Migration?","category":"agentic","description":"Tests whether coding agents can complete long-horizon, whole-repository stack migrations while preserving the original program's behavior.","paperUrl":"https://arxiv.org/abs/2608.23564","paperTitle":"SWE Refactor Bench: Can Coding Agents Complete a Long-Horizon, Whole-Repository Stack Migration?","authors":"Deyao Hong, Yizhe Chi, Wenyi Li, Xiaoqiu Wang, Mingju Gao, Kaisen Yang, Bingxiang He, Youjie Zheng, Calvin Xiao, Qinhuai Na","year":"2026","tasks":"20 whole-repository stack migrations","format":"Migration audit, frozen behavioral checks, and agentic verification","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","score":23}]},{"key":"spider2Lite","name":"Spider 2.0-Lite","fullName":"Spider 2.0-Lite","category":"coding","description":"A text-to-SQL benchmark over realistic warehouse-scale schemas, reported by Interfaze for model comparison.","paperUrl":"https://github.com/xlang-ai/Spider2","paperTitle":"Spider 2.0: Evaluating Language Models on Real-World Enterprise Text-to-SQL Workflows","authors":"Spider 2.0 authors","year":"2024","tasks":"Text-to-SQL queries","format":"Execution accuracy","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"interfaze-beta","name":"Interfaze Beta","score":52.9}]},{"key":"spreadsheetBench2","name":"SpreadsheetBench 2","fullName":"SpreadsheetBench 2","category":"agentic","description":"A spreadsheet-focused benchmark for agentic analysis and editing workflows.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI","year":"2026","tasks":"Spreadsheet analysis and editing tasks","format":"Agent task-completion score","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"kimi-k3","name":"Kimi K3","score":34.8}]},{"key":"terminalBenchScience6xVerifier","name":"Terminal-Bench-Science 0.1 (6x verifier timeout)","fullName":"Terminal-Bench-Science 0.1 with sixfold verifier timeout","category":"agentic","description":"Google-run scientific workflow evaluation using a sixfold verifier timeout to address verification timeouts.","paperUrl":"https://deepmind.google/models/evals-methodology/gemini-4-argon","paperTitle":"Gemini 4 Argon evaluation methodology","authors":"Google DeepMind","year":"2026","tasks":"70 expert-curated scientific research workflows","format":"Resolution rate across 3 trials per task","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"gemini-4-argon","name":"Gemini 4 Argon","score":57.6}]},{"key":"vibeV2","name":"VIBE V2","fullName":"VIBE V2","category":"coding","description":"A display-only MiniMax provider benchmark for end-to-end coding-agent and product-building tasks.","paperUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M3","paperTitle":"MiniMax M3 model card","authors":"MiniMax","year":"2026","tasks":"End-to-end coding-agent tasks","format":"Task success rate","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"minimax-m3","name":"MiniMax M3","score":50.1}]},{"key":"vibePro","name":"VIBE-Pro","fullName":"VIBE-Pro","category":"coding","description":"A repo-level code generation and full-project delivery benchmark spanning web, mobile, and simulation-style implementation tasks.","paperUrl":"https://www.minimax.io/news/minimax-m27-en","paperTitle":"MiniMax M2.7: Early Echoes of Self-Evolution","authors":"MiniMax","year":"2026","tasks":"Full project delivery tasks","format":"Repository-level implementation benchmark","decimals":null,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"minimax-m2-7","name":"MiniMax M2.7","score":55.6}]},{"key":"wmt26","name":"WMT26","fullName":"WMT26 General Machine Translation (all languages)","category":"multilingual","description":"Machine-translation quality on the WMT26 general translation task, averaged across all evaluated languages and scored 0-100 by an LLM judge.","paperUrl":"https://www2.statmt.org/wmt26/translation-task.html","paperTitle":"WMT26 General Machine Translation Task","authors":"WMT organizers","year":"2026","tasks":"General-domain translation across 50+ languages (32 high-resource plus 18 additional)","format":"0-100 judged translation quality, averaged across languages","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"north-small-translate-1-0","name":"North Small Translate","score":83.6}]},{"key":"webArena","name":"WebArena","fullName":"WebArena Web Agent Benchmark","category":"agentic","description":"WebArena tests whether a browser-agent system can complete 812 long-horizon tasks inside self-hosted replicas of functional websites. It checks the requested end state, so a result reflects the model, agent scaffold, browser interface, action budget, and evaluator together—not the base model alone.","paperUrl":"https://arxiv.org/abs/2307.13854","paperTitle":"WebArena: A Realistic Web Environment for Building Autonomous Agents","authors":"Shuyan Zhou, Frank F. Xu, Hao Zhu, Xuhui Zhou, Robert Lo, Abishek Sridhar, Xianyi Cheng, Tianyue Ou, Yonatan Bisk, Daniel Fried, Uri Alon, Graham Neubig","year":"2024","tasks":"812 long-horizon browser tasks","format":"End-state task success","decimals":null,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"nemotron-3-5-bolt-74b-a9b","name":"Nemotron 3.5 Bolt 74B-A9B","score":50.86}]},{"key":"webVoyager","name":"WebVoyager","fullName":"WebVoyager","category":"agentic","description":"A browser-agent benchmark for completing multi-step workflows on live websites.","paperUrl":"https://docs.z.ai/guides/vlm/glm-5v-turbo","paperTitle":"GLM-5V-Turbo","authors":"Z.AI","year":"2026","tasks":"Live website workflows","format":"Interactive browser-agent evaluation","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"fara-1-5-27b","name":"Fara1.5-27B","score":89.3}]},{"key":"worldVqaForceAnswer","name":"WorldVQA ForceAnswer","fullName":"WorldVQA ForceAnswer","category":"multimodalGrounded","description":"A forced-answer WorldVQA variant for atomic visual world knowledge.","paperUrl":"https://www.kimi.com/blog/kimi-k3","paperTitle":"Kimi K3: Open Frontier Intelligence","authors":"Moonshot AI / WorldVQA authors","year":"2026","tasks":"Atomic visual world-knowledge questions","format":"Forced-answer visual QA","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"kimi-k3","name":"Kimi K3","score":51}]},{"key":"olmOcr","name":"olmOCR","fullName":"olmOCR-Bench","category":"multimodalGrounded","description":"An end-to-end document understanding benchmark over long, layout-rich PDFs with tables, equations, headers, footnotes, and multi-column flows.","paperUrl":"https://github.com/allenai/olmocr/tree/main/olmocr/bench","paperTitle":"olmOCR-Bench","authors":"Allen Institute for AI","year":"2025","tasks":"Layout-rich PDF understanding","format":"Mean accuracy","decimals":1,"modelCount":1,"lowerIsBetter":false,"top":[{"slug":"interfaze-beta","name":"Interfaze Beta","score":85.7}]}]