{ "schema_version": 1, "collected_at": "2026-11-02T07:12:02.444309+00:00", "benchmarks": [ { "id": "swe-bench-verified", "name": "SWE-bench Verified", "group": "编程", "unit": "%", "metric": "Resolved", "protocol": { "en": "500 Verified tasks; mini-SWE-agent, one attempt. Agent version is shown for each result.", "zh": "500 项 Verified 任务；mini-SWE-agent，单次尝试。各条成绩注明 Agent 版本。" }, "source": { "name": "SWE-bench", "url": "https://www.swebench.com/", "data_url": "https://www.swebench.com/", "retrieved_at": "2026-11-01T16:45:14.749325+00:00", "sha256": "c862011a4ee1d1a7199fad6ca0905fa5c64f58216f7217ac1eddf37f28788433", "selector": "Verified; mini-SWE-agent; System: Attempts - 1; resolved / 100", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "f73a2bb3f914d4f31429", "model": "Claude 4.5 Opus (high)", "organization": "Anthropic", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.768, "raw_score": 76.8, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_claude-4-5-opus-high", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "a880201db8e29235cffc", "model": "Gemini 3 Flash (high)", "organization": "Google DeepMind", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.758, "raw_score": 75.8, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_gemini-3-flash-high", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "aad9f1fc9679401226e2", "model": "MiniMax M2.5 (high)", "organization": "Minimax", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.758, "raw_score": 75.8, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_minimax-2-5-high", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "b077a30860c6936d9861", "model": "Claude 4.6 Opus", "organization": "Anthropic", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.7559999999999999, "raw_score": 75.6, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_claude-4-6-opus", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "0840be8dd13de47b9a25", "model": "Claude 4.5 Opus (20251101) (medium)", "organization": "Anthropic", "configuration": "mini-SWE-agent 1.16.0; 1 attempt", "score": 0.7440000000000001, "raw_score": 74.4, "raw_scale": 100, "source_record": "20251124_mini-v1.16.0_claude-opus-4-5-20251101", "source_date": "2025-11-24", "checked_by_publisher": true }, { "id": "15ac3ef241754e069492", "model": "Gemini 3 Pro Preview (2025-11-18)", "organization": "Google DeepMind", "configuration": "mini-SWE-agent 1.15.0; 1 attempt", "score": 0.742, "raw_score": 74.2, "raw_scale": 100, "source_record": "20251118_mini-v1.15.0_gemini-3-pro-preview-20251118", "source_date": "2025-11-18", "checked_by_publisher": true }, { "id": "e49605ad5747553b8b16", "model": "GLM 5 (high)", "organization": "Z-AI", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.728, "raw_score": 72.8, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_glm-5-high", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "d6a136ecd88e83590ffe", "model": "GPT 5.2 (high)", "organization": "OpenAI", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.728, "raw_score": 72.8, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_gpt-5-2-high", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "2e070986eebe483b6fa4", "model": "GPT 5.2 Codex", "organization": "OpenAI", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.728, "raw_score": 72.8, "raw_scale": 100, "source_record": "20260219_mini-v2.0.0_gpt-5-2-codex", "source_date": "2026-02-19", "checked_by_publisher": null }, { "id": "556e9e1ce4be2eaa50f6", "model": "GPT 5.2 (2025-12-11) (high)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.17.2; 1 attempt", "score": 0.718, "raw_score": 71.8, "raw_scale": 100, "source_record": "20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high", "source_date": "2025-12-11", "checked_by_publisher": null }, { "id": "ec1e7f6754efe9265270", "model": "Claude 4.5 Sonnet (high)", "organization": "Anthropic", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.7140000000000001, "raw_score": 71.4, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_claude-4-5-sonnet-high", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "63bda662d33e1ddaf707", "model": "Kimi K2.5 (high)", "organization": "Moonshot AI", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.708, "raw_score": 70.8, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_kimi-k2-5-high", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "2378af15c0e1ef069e50", "model": "Claude 4.5 Sonnet (20250929)", "organization": "Anthropic", "configuration": "mini-SWE-agent 1.13.3; 1 attempt", "score": 0.706, "raw_score": 70.6, "raw_scale": 100, "source_record": "20250929_mini-v1.13.3_sonnet-4-5-20250929", "source_date": "2025-09-29", "checked_by_publisher": true }, { "id": "9fa6b53e8ea7f43f7900", "model": "DeepSeek V3.2 (high)", "organization": "DeepSeek", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.7, "raw_score": 70.0, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_deepseek-3-2-high", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "babe2b82ca849ab569a6", "model": "Gemini 3 Pro", "organization": "Google DeepMind", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.696, "raw_score": 69.6, "raw_scale": 100, "source_record": "20260226_mini-v2.0.0_gemini-3-pro-high", "source_date": "2026-02-26", "checked_by_publisher": null }, { "id": "d28b57598238e0bcfb17", "model": "GPT 5.2 (2025-12-11)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.17.2; 1 attempt", "score": 0.69, "raw_score": 69.0, "raw_scale": 100, "source_record": "20251211_mini-v1.17.2_gpt-5.2-2025-12-11", "source_date": "2025-12-11", "checked_by_publisher": null }, { "id": "65a5780b3bd4690d940f", "model": "Claude 4 Opus (20250514)", "organization": "Anthropic", "configuration": "mini-SWE-agent 1.0.0; 1 attempt", "score": 0.6759999999999999, "raw_score": 67.6, "raw_scale": 100, "source_record": "20250802_mini-v1.0.0_claude-4-opus-20250514", "source_date": "2025-08-02", "checked_by_publisher": true }, { "id": "458b62d4a06cb2636eb8", "model": "Claude 4.5 Haiku (high)", "organization": "Anthropic", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.6659999999999999, "raw_score": 66.6, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_claude-4-5-haiku-high", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "3e3b71bc126738e83224", "model": "GPT 5.1 (2025-11-13) (medium)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.15.0; 1 attempt", "score": 0.66, "raw_score": 66.0, "raw_scale": 100, "source_record": "20251120_mini-v1.15.0_gpt-5.1-2025-11-13", "source_date": "2025-11-20", "checked_by_publisher": null }, { "id": "3d043b338b090b6e71a5", "model": "GPT 5.1 Codex (medium)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.16.0; 1 attempt", "score": 0.66, "raw_score": 66.0, "raw_scale": 100, "source_record": "20251124_mini-v1.16.0_gpt-5.1-codex", "source_date": "2025-11-24", "checked_by_publisher": null }, { "id": "ff6cc7977cc397b31422", "model": "GPT 5 (2025-08-07) (medium)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.7.0; 1 attempt", "score": 0.65, "raw_score": 65.0, "raw_scale": 100, "source_record": "20250807_mini-v1.7.0_gpt-5", "source_date": "2025-08-07", "checked_by_publisher": true }, { "id": "bb21450fac3124ec799b", "model": "Claude 4 Sonnet (20250514)", "organization": "Anthropic", "configuration": "mini-SWE-agent 1.0.0; 1 attempt", "score": 0.6493000000000001, "raw_score": 64.93, "raw_scale": 100, "source_record": "20250726_mini-v1.0.0_claude-sonnet-4-20250514", "source_date": "2025-07-26", "checked_by_publisher": true }, { "id": "dea2258d6c32d4d35984", "model": "Kimi K2 Thinking", "organization": "Moonshot AI", "configuration": "mini-SWE-agent 1.17.2; 1 attempt", "score": 0.634, "raw_score": 63.4, "raw_scale": 100, "source_record": "20251210_mini-v1.17.2_kimi-k2-thinking", "source_date": "2025-12-10", "checked_by_publisher": true }, { "id": "df561e5e19ae399fb943", "model": "MiniMax M2", "organization": "Minimax", "configuration": "mini-SWE-agent 1.17.0; 1 attempt", "score": 0.61, "raw_score": 61.0, "raw_scale": 100, "source_record": "20251124_mini-v1.17.0_minimax-m2", "source_date": "2025-11-24", "checked_by_publisher": true }, { "id": "37bb967698aac3cbfb4b", "model": "DeepSeek V3.2 Reasoner", "organization": "deepseek", "configuration": "mini-SWE-agent 1.17.1; 1 attempt", "score": 0.6, "raw_score": 60.0, "raw_scale": 100, "source_record": "20251201_mini-v1.17.1_deepseek-v3.2-reasoner", "source_date": "2025-12-01", "checked_by_publisher": true }, { "id": "d3cbf1c8b895ecac24a9", "model": "GPT 5 mini (2025-08-07) (medium)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.7.0; 1 attempt", "score": 0.598, "raw_score": 59.8, "raw_scale": 100, "source_record": "20250807_mini-v1.7.0_gpt-5-mini", "source_date": "2025-08-07", "checked_by_publisher": true }, { "id": "0a6f5a221a7e982481e0", "model": "o3 (2025-04-16)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.0.0; 1 attempt", "score": 0.584, "raw_score": 58.4, "raw_scale": 100, "source_record": "20250726_mini-v1.0.0_o3-2025-04-16", "source_date": "2025-07-26", "checked_by_publisher": true }, { "id": "91a1945f00ecc1d313b4", "model": "Devstral Small (2512)", "organization": "mistral", "configuration": "mini-SWE-agent 1.17.2; 1 attempt", "score": 0.564, "raw_score": 56.4, "raw_scale": 100, "source_record": "20251209_mini-v1.17.2_devstral-small-2512", "source_date": "2025-12-09", "checked_by_publisher": true }, { "id": "a41e43c56591df664ae0", "model": "GPT 5 mini", "organization": "OpenAI", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.562, "raw_score": 56.2, "raw_scale": 100, "source_record": "20260217_mini-v2.0.0_gpt-5-mini", "source_date": "2026-02-17", "checked_by_publisher": null }, { "id": "3e296d2b022465a40bcd", "model": "GLM 4.6 (T=1)", "organization": "Z.ai", "configuration": "mini-SWE-agent 1.17.1; 1 attempt", "score": 0.5539999999999999, "raw_score": 55.4, "raw_scale": 100, "source_record": "20251201_mini-v1.17.1_glm-4.6", "source_date": "2025-12-01", "checked_by_publisher": true }, { "id": "96249723cf5095b44f13", "model": "Qwen3-Coder 480B/A35B Instruct", "organization": "Qwen", "configuration": "mini-SWE-agent 1.0.0; 1 attempt", "score": 0.5539999999999999, "raw_score": 55.4, "raw_scale": 100, "source_record": "20250802_mini-v1.0.0_qwen3-coder-480b-a35b-instruct", "source_date": "2025-08-02", "checked_by_publisher": true }, { "id": "884be3105858dfd47769", "model": "GLM 4.5 (2025-08-22)", "organization": "Z.ai", "configuration": "mini-SWE-agent 1.9.1; 1 attempt", "score": 0.542, "raw_score": 54.2, "raw_scale": 100, "source_record": "20250822_mini-v1.9.1_glm-4.5", "source_date": "2025-08-22", "checked_by_publisher": true }, { "id": "0a0e4d3807af6fce7236", "model": "Devstral (2512)", "organization": "mistral", "configuration": "mini-SWE-agent 1.17.2; 1 attempt", "score": 0.5379999999999999, "raw_score": 53.8, "raw_scale": 100, "source_record": "20251209_mini-v1.17.2_devstral-2512", "source_date": "2025-12-09", "checked_by_publisher": true }, { "id": "2d92300bf68f6bf99359", "model": "Gemini 2.5 Pro (2025-05-06)", "organization": "Google DeepMind", "configuration": "mini-SWE-agent 1.0.0; 1 attempt", "score": 0.536, "raw_score": 53.6, "raw_scale": 100, "source_record": "20250726_mini-v1.0.0_gemini-2.5-pro", "source_date": "2025-07-26", "checked_by_publisher": true }, { "id": "e441da736d0d79a0b47d", "model": "Claude 3.7 Sonnet (20250219)", "organization": "Anthropic", "configuration": "mini-SWE-agent 0.0.0; 1 attempt", "score": 0.528, "raw_score": 52.8, "raw_scale": 100, "source_record": "20250720_mini-v0.0.0-claude-3-7-sonnet-20250219", "source_date": "2025-07-20", "checked_by_publisher": true }, { "id": "3d34dec2d986bc4bd217", "model": "o4-mini (2025-04-16)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.0.0; 1 attempt", "score": 0.45, "raw_score": 45.0, "raw_scale": 100, "source_record": "20250726_mini-v1.0.0_o4-mini-2025-04-16", "source_date": "2025-07-26", "checked_by_publisher": true }, { "id": "6f50253d5875d4519784", "model": "Kimi K2 Instruct", "organization": "Moonshot AI", "configuration": "mini-SWE-agent 1.7.0; 1 attempt", "score": 0.43799999999999994, "raw_score": 43.8, "raw_scale": 100, "source_record": "20250807_mini-v1.7.0_kimi-k2-instruct", "source_date": "2025-08-07", "checked_by_publisher": true }, { "id": "2b54524e6c6e41951b84", "model": "GPT 4.1 (2025-04-14)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.0.0; 1 attempt", "score": 0.3958, "raw_score": 39.58, "raw_scale": 100, "source_record": "20250726_mini-v1.0.0_gpt-4.1-2025-04-14", "source_date": "2025-07-26", "checked_by_publisher": true }, { "id": "5b66484f4499f4578160", "model": "GPT 5 nano (2025-08-07) (medium)", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.7.0; 1 attempt", "score": 0.348, "raw_score": 34.8, "raw_scale": 100, "source_record": "20250807_mini-v1.7.0_gpt-5-nano", "source_date": "2025-08-07", "checked_by_publisher": true }, { "id": "8c7d326c4165bafb8051", "model": "Gemini 2.5 Flash (2025-04-17)", "organization": "Google DeepMind", "configuration": "mini-SWE-agent 1.0.0; 1 attempt", "score": 0.2873, "raw_score": 28.73, "raw_scale": 100, "source_record": "20250726_mini-v1.0.0_gemini-2.5-flash", "source_date": "2025-07-26", "checked_by_publisher": true }, { "id": "c19e0aed51e8766ab99f", "model": "gpt-oss-120b", "organization": "OpenAI", "configuration": "mini-SWE-agent 1.7.0; 1 attempt", "score": 0.26, "raw_score": 26.0, "raw_scale": 100, "source_record": "20250807_mini-v1.7.0_gpt-oss-120b", "source_date": "2025-08-07", "checked_by_publisher": true }, { "id": "3fc3e9384f7a66f98b78", "model": "GPT 4.1 mini (2025-04-14)", "organization": "OpenAI", "configuration": "mini-SWE-agent 0.0.0; 1 attempt", "score": 0.2394, "raw_score": 23.94, "raw_scale": 100, "source_record": "20250720_mini-v0.0.0_gpt-4.1-mini-2025-04-14", "source_date": "2025-07-20", "checked_by_publisher": true }, { "id": "7bce4c273c14f4c3a28a", "model": "GPT 4o (2024-11-20)", "organization": "OpenAI", "configuration": "mini-SWE-agent 0.0.0; 1 attempt", "score": 0.2162, "raw_score": 21.62, "raw_scale": 100, "source_record": "20250720_mini-v0.0.0-gpt-4o-2024-11-20", "source_date": "2025-07-20", "checked_by_publisher": true }, { "id": "e93f9d0bde3a1d3f050e", "model": "Llama 4 Maverick Instruct", "organization": "Meta", "configuration": "mini-SWE-agent 0.0.0; 1 attempt", "score": 0.2104, "raw_score": 21.04, "raw_scale": 100, "source_record": "20250720_mini-v0.0.0-Llama-4-Maverick-17B-Instruct", "source_date": "2025-07-20", "checked_by_publisher": true }, { "id": "2a0a6351df234f6fff8c", "model": "Gemini 2.0 Flash", "organization": "Google DeepMind", "configuration": "mini-SWE-agent 0.0.0; 1 attempt", "score": 0.1352, "raw_score": 13.52, "raw_scale": 100, "source_record": "20250726_mini-v1.0.0_gemini-2.0-flash", "source_date": "2025-07-26", "checked_by_publisher": true }, { "id": "f48285f643b9318d3ad6", "model": "Llama 4 Scout Instruct", "organization": "Meta", "configuration": "mini-SWE-agent 0.0.0; 1 attempt", "score": 0.0906, "raw_score": 9.06, "raw_scale": 100, "source_record": "20250720_mini-v0.0.0-Llama-4-Scout-17B-Instruct", "source_date": "2025-07-20", "checked_by_publisher": true }, { "id": "aea2895aae5a352c51d5", "model": "Qwen2.5-Coder 32B Instruct", "organization": "Qwen", "configuration": "mini-SWE-agent 1.0.0; 1 attempt", "score": 0.09, "raw_score": 9.0, "raw_scale": 100, "source_record": "20250803_mini-v1.0.0_qwen2-5-coder-32b-instruct", "source_date": "2025-08-03", "checked_by_publisher": true } ] }, { "id": "arc-agi-2", "name": "ARC-AGI-2", "group": "知识与推理", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Semi-private evaluation; published LLM and reasoning configurations. Human baselines and competition systems are excluded.", "zh": "半私有评测集；保留官方公开的大模型及推理配置成绩，不含人类基线和竞赛专用系统。" }, "source": { "name": "ARC Prize", "url": "https://arcprize.org/leaderboard", "data_url": "https://arcprize.org/media/data/leaderboard/v2.json", "retrieved_at": "2026-11-02T05:57:30.777829+00:00", "sha256": "dab3a27ecc57b5257260fd3123197ba463f11af569c97f582e8cd5bf153a6ef9", "selector": "v2_Semi_Private; display=true; Base LLM / CoT / CoT + Synthesis / Refinement", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "88f48429bd39d2492872", "model": "GPT-6 Astra (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.95, "raw_score": 0.95, "raw_scale": 1, "source_record": "openai-gpt-6-astra-max", "cost_per_task": 1.1194680833333335 }, { "id": "97f2d39744d27a43065d", "model": "GPT-6.1 Sol (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.9416666666666667, "raw_score": 0.9416666666666667, "raw_scale": 1, "source_record": "openai-gpt-6-1-sol-max", "cost_per_task": 0.25374978333333337 }, { "id": "0d653b3d3182ef6c2012", "model": "Claude Opus 5.5 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.9333333333333333, "raw_score": 0.9333333333333333, "raw_scale": 1, "source_record": "anthropic-claude-opus-5-5-high", "cost_per_task": 0.4078964666666666 }, { "id": "960d3aa0d1d58c8001df", "model": "GPT-6 Astra (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.9333333333333333, "raw_score": 0.9333333333333333, "raw_scale": 1, "source_record": "openai-gpt-6-astra-xhigh", "cost_per_task": 0.8286855833333331 }, { "id": "90d8524ae331acf31f72", "model": "Claude Opus 5.5 (XHigh)", "organization": "Anthropic", "configuration": "CoT", "score": 0.925, "raw_score": 0.925, "raw_scale": 1, "source_record": "anthropic-claude-opus-5-5-xhigh", "cost_per_task": 0.6709066333333333 }, { "id": "be551e1a6f34d6d063c9", "model": "GPT-5.6 Sol (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.925, "raw_score": 0.925, "raw_scale": 1, "source_record": "openai-gpt-5-6-sol-max", "cost_per_task": 1.44 }, { "id": "d29dec4f370bc4a6c95c", "model": "GPT-6 Astra (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.9208333333333333, "raw_score": 0.9208333333333333, "raw_scale": 1, "source_record": "openai-gpt-6-astra-high", "cost_per_task": 0.6680793333333332 }, { "id": "8782166651b943a59500", "model": "GPT-6 Astra (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.9208333333333333, "raw_score": 0.9208333333333333, "raw_scale": 1, "source_record": "openai-gpt-6-astra-medium", "cost_per_task": 0.4800926666666667 }, { "id": "d34d009549c9852bbd68", "model": "Claude Opus 5.5 (Max)", "organization": "Anthropic", "configuration": "CoT", "score": 0.9166666666666666, "raw_score": 0.9166666666666666, "raw_scale": 1, "source_record": "anthropic-claude-opus-5-5-max", "cost_per_task": 1.8517409666666667 }, { "id": "5b50cdc5208933289fd0", "model": "GPT-6.1 Sol (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.9166666666666666, "raw_score": 0.9166666666666666, "raw_scale": 1, "source_record": "openai-gpt-6-1-sol-high", "cost_per_task": 0.13468670000000002 }, { "id": "8b07e0fd8e99bdeb3f66", "model": "GPT-6.1 Sol (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.9166666666666666, "raw_score": 0.9166666666666666, "raw_scale": 1, "source_record": "openai-gpt-6-1-sol-xhigh", "cost_per_task": 0.17808970000000007 }, { "id": "6d70c2b5335d2eabf780", "model": "Gemini 4 Argon (High)", "organization": "Google", "configuration": "CoT", "score": 0.9125, "raw_score": 0.9125, "raw_scale": 1, "source_record": "google-gemini-4-argon-high", "cost_per_task": 0.3248761333333333 }, { "id": "3bf0b552033ccc79a746", "model": "Claude Opus 5 (Max)", "organization": "Anthropic", "configuration": "CoT", "score": 0.9042, "raw_score": 0.9042, "raw_scale": 1, "source_record": "anthropic-claude-opus-5-max", "cost_per_task": 2.06 }, { "id": "02e7105417a2f5a3dbdc", "model": "Claude Fable 5.1 (Max)", "organization": "Anthropic", "configuration": "CoT", "score": 0.9, "raw_score": 0.9, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-1-max", "cost_per_task": 4.492348833333334 }, { "id": "3702690a9a14e12657aa", "model": "Claude Fable 5.1 (XHigh)", "organization": "Anthropic", "configuration": "CoT", "score": 0.9, "raw_score": 0.9, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-1-xhigh", "cost_per_task": 3.120216916666669 }, { "id": "a0a23fbcdc116480e346", "model": "GPT-5.6 Sol (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.9, "raw_score": 0.9, "raw_scale": 1, "source_record": "openai-gpt-5-6-sol-xhigh", "cost_per_task": 1.04 }, { "id": "bd1c0c7c32c0b3d290f1", "model": "GPT-6 Sol (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8958333333333334, "raw_score": 0.8958333333333334, "raw_scale": 1, "source_record": "openai-gpt-6-sol-max", "cost_per_task": 0.43889828333333336 }, { "id": "73b450598adcf4e256e6", "model": "Claude Fable 5 (Max)", "organization": "Anthropic", "configuration": "CoT", "score": 0.8916666666666667, "raw_score": 0.8916666666666667, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-max", "cost_per_task": 5.449076000000001 }, { "id": "44f9b77a07f6982a8f55", "model": "Gemini 3.8 Flash (High)", "organization": "Google", "configuration": "CoT", "score": 0.8916666666666667, "raw_score": 0.8916666666666667, "raw_scale": 1, "source_record": "google-gemini-3-8-flash-high", "cost_per_task": 0.39972941250000005 }, { "id": "2eadfdf622a2fefd0da0", "model": "Claude Fable 5.1 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.8875, "raw_score": 0.8875, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-1-high", "cost_per_task": 1.6673061666666666 }, { "id": "785e442d385a92780369", "model": "Claude Fable 5 (XHigh)", "organization": "Anthropic", "configuration": "CoT", "score": 0.8833333333333333, "raw_score": 0.8833333333333333, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-xhigh", "cost_per_task": 3.0043564166666665 }, { "id": "556b644f6d06587817da", "model": "Claude Opus 5 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.8833, "raw_score": 0.8833, "raw_scale": 1, "source_record": "anthropic-claude-opus-5-high", "cost_per_task": 1.45 }, { "id": "557ad1ac5d6291a3d2cf", "model": "Claude Fable 5 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.875, "raw_score": 0.875, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-high", "cost_per_task": 2.4358764166666664 }, { "id": "0a699113a6a9e7336fbb", "model": "Claude Opus 5.5 (Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 0.875, "raw_score": 0.875, "raw_scale": 1, "source_record": "anthropic-claude-opus-5-5-medium", "cost_per_task": 0.3369188 }, { "id": "275929d7ea1625be8603", "model": "Gemini 4 Argon (Medium)", "organization": "Google", "configuration": "CoT", "score": 0.8708333333333333, "raw_score": 0.8708333333333333, "raw_scale": 1, "source_record": "google-gemini-4-argon-medium", "cost_per_task": 0.13732313333333326 }, { "id": "0fbea5791eb486f0cc69", "model": "GPT-6.1 Sol (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8666666666666667, "raw_score": 0.8666666666666667, "raw_scale": 1, "source_record": "openai-gpt-6-1-sol-medium", "cost_per_task": 0.10037736666666668 }, { "id": "17e7c24c3fab5e5ef054", "model": "Claude Fable 5.1 (Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 0.8625, "raw_score": 0.8625, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-1-medium", "cost_per_task": 1.2239411666666673 }, { "id": "411d002334d2986b5391", "model": "GPT-5.6 Sol (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8542, "raw_score": 0.8542, "raw_scale": 1, "source_record": "openai-gpt-5-6-sol-high", "cost_per_task": 0.74 }, { "id": "8bb37c7f90fbb8c18f36", "model": "GPT-6 Astra (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8541666666666666, "raw_score": 0.8541666666666666, "raw_scale": 1, "source_record": "openai-gpt-6-astra-low", "cost_per_task": 0.4162418333333334 }, { "id": "ef7fb9a8ab21ab3b2e02", "model": "GPT-5.5 (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.85, "raw_score": 0.85, "raw_scale": 1, "source_record": "gpt-5-5-2026-04-22-thinking-xhigh", "cost_per_task": 1.87 }, { "id": "bf35eaea1608c619d15b", "model": "Gemini 3.7 Flash (High)", "organization": "Google", "configuration": "CoT", "score": 0.8458333333333333, "raw_score": 0.8458333333333333, "raw_scale": 1, "source_record": "google-gemini-3-7-flash-high", "cost_per_task": 0.24944951875000002 }, { "id": "9be11fa4258a4683fcf9", "model": "GPT-5.5 Pro (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8458, "raw_score": 0.8458, "raw_scale": 1, "source_record": "gpt-5-5-pro-2026-04-23-high", "cost_per_task": 10.51 }, { "id": "a86c0bbc28a816c66091", "model": "Gemini 3 Deep Think (2/26)", "organization": "Google", "configuration": "CoT", "score": 0.8458, "raw_score": 0.8458, "raw_scale": 1, "source_record": "gemini_3_deep_think_022026", "cost_per_task": 13.6213 }, { "id": "564823075336d90a3aa7", "model": "GPT-5.5 Pro (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8416, "raw_score": 0.8416, "raw_scale": 1, "source_record": "gpt-5-5-pro-2026-04-23-xhigh", "cost_per_task": 10.76 }, { "id": "11fdc7e72e882264b672", "model": "GPT-5.6 Terra (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.839, "raw_score": 0.839, "raw_scale": 1, "source_record": "openai-gpt-5-6-terra-max", "cost_per_task": 1.09 }, { "id": "7a7cec914c398dd0468a", "model": "Claude Sonnet 5.5 (XHigh)", "organization": "Anthropic", "configuration": "CoT", "score": 0.8388888888888889, "raw_score": 0.8388888888888889, "raw_scale": 1, "source_record": "anthropic-claude-sonnet-5-5-xhigh", "cost_per_task": 0.5461994833333332 }, { "id": "d7dc814219da3b2b8da5", "model": "GPT-5.4 Pro (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8333, "raw_score": 0.8333, "raw_scale": 1, "source_record": "gpt-5-4-pro-xhigh", "cost_per_task": 16.41 }, { "id": "d322a41ea58dc0e6c7ab", "model": "GPT-5.5 (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8333, "raw_score": 0.8333, "raw_scale": 1, "source_record": "gpt-5-5-2026-04-22-thinking-high", "cost_per_task": 1.45 }, { "id": "3ca161cae85865931ef9", "model": "Gemini 3.8 Flash (Medium)", "organization": "Google", "configuration": "CoT", "score": 0.8291666666666667, "raw_score": 0.8291666666666667, "raw_scale": 1, "source_record": "google-gemini-3-8-flash-medium", "cost_per_task": 0.2477876749999999 }, { "id": "978707fd71acf0edabb3", "model": "Claude Fable 5 (Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 0.825, "raw_score": 0.825, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-medium", "cost_per_task": 1.9269780833333334 }, { "id": "5dcbffe6dafc5d6792d8", "model": "Claude Fable 5.1 (Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.7833333333333333, "raw_score": 0.7833333333333333, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-1-low", "cost_per_task": 0.9523265833333336 }, { "id": "f1b0a937ae53193080ba", "model": "DeepSeek V4.1 Flash 1017 (Max)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.7805555555555556, "raw_score": 0.7805555555555556, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-1017-max", "cost_per_task": 0.163116895 }, { "id": "009e7fba6bdf4864bbe0", "model": "GPT-6 Sol (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.7805555555555556, "raw_score": 0.7805555555555556, "raw_scale": 1, "source_record": "openai-gpt-6-sol-xhigh", "cost_per_task": 0.27722933333333344 }, { "id": "b8487af4bea01a4bd35f", "model": "Gemini 3.8 Flash (Low)", "organization": "Google", "configuration": "CoT", "score": 0.775, "raw_score": 0.775, "raw_scale": 1, "source_record": "google-gemini-3-8-flash-low", "cost_per_task": 0.15445401874999998 }, { "id": "9a8a8310835e24f5b213", "model": "Gemini 3.1 Pro (Preview)", "organization": "Google", "configuration": "CoT", "score": 0.7708, "raw_score": 0.7708, "raw_scale": 1, "source_record": "gemini-3-1-pro-preview", "cost_per_task": 0.9622 }, { "id": "e32b3d91ea67cdff994f", "model": "Dots3-Note Preview (Max)", "organization": "Dots Studio", "configuration": "CoT", "score": 0.7680555555555556, "raw_score": 0.7680555555555556, "raw_scale": 1, "source_record": "dots-studio-dots3-note-prev-max", "cost_per_task": 0.07665574466666672 }, { "id": "f8b733c146e7c8714a8f", "model": "Claude Fable 5 (Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.7680555555555555, "raw_score": 0.7680555555555555, "raw_scale": 1, "source_record": "anthropic-claude-fable-5-low", "cost_per_task": 1.4489280833333338 }, { "id": "19a9fdeabdc46906a982", "model": "GPT-6.1 Sol (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.7666666666666667, "raw_score": 0.7666666666666667, "raw_scale": 1, "source_record": "openai-gpt-6-1-sol-low", "cost_per_task": 0.08553803333333333 }, { "id": "b7154601fd0ce85420d5", "model": "Claude 4.7 (Max)", "organization": "Anthropic", "configuration": "CoT", "score": 0.7583, "raw_score": 0.7583, "raw_scale": 1, "source_record": "claude-4-7-max", "cost_per_task": 7.43 }, { "id": "b63f029908208cf65cc4", "model": "GPT-5.6 Terra (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.7417, "raw_score": 0.7417, "raw_scale": 1, "source_record": "openai-gpt-5-6-terra-xhigh", "cost_per_task": 0.69 }, { "id": "be942b73a500724d24eb", "model": "GPT-5.4 (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.7395, "raw_score": 0.7395, "raw_scale": 1, "source_record": "gpt-5-4-xhigh", "cost_per_task": 1.52 }, { "id": "953750815a3ba5a9119f", "model": "DeepSeek V4.1 Flash 1017 (Low)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.7361111111111112, "raw_score": 0.7361111111111112, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-1017-low", "cost_per_task": 0.133540355 }, { "id": "9be3411ecd59e8477583", "model": "DeepSeek V4.1 Flash (Max)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.7291666666666666, "raw_score": 0.7291666666666666, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-max", "cost_per_task": 0.12938378500000003 }, { "id": "aa7eae2614fa6cfce097", "model": "GPT-5.2 (Refine.)", "organization": "OpenAI", "configuration": "Refinement; Johan Land", "score": 0.729, "raw_score": 0.729, "raw_scale": 1, "source_record": "johan_land_gpt_5_2_refine", "cost_per_task": 38.99 }, { "id": "c1c62af6d7218cb6d440", "model": "Claude Opus 4.8 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.7208, "raw_score": 0.7208, "raw_scale": 1, "source_record": "anthropic-opus-4-8-high", "cost_per_task": 2.7425 }, { "id": "11e219f9126bd57b665f", "model": "Gemini 3.5 Flash (High)", "organization": "Google", "configuration": "CoT", "score": 0.7208, "raw_score": 0.7208, "raw_scale": 1, "source_record": "gemini-3-5-flash-high", "cost_per_task": 0.8503 }, { "id": "beababfc7c1ee6b47a1a", "model": "Claude Opus 4.8 (Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 0.7167, "raw_score": 0.7167, "raw_scale": 1, "source_record": "anthropic-opus-4-8-medium", "cost_per_task": 2.3896 }, { "id": "af89f18a0c11ffdd2e32", "model": "GPT-5.5 (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.7042, "raw_score": 0.7042, "raw_scale": 1, "source_record": "gpt-5-5-2026-04-22-thinking-medium", "cost_per_task": 0.86 }, { "id": "a4484f15e17532c6e92b", "model": "Claude Opus 5.5 (Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.701388888888889, "raw_score": 0.701388888888889, "raw_scale": 1, "source_record": "anthropic-claude-opus-5-5-low", "cost_per_task": 0.24127179999999998 }, { "id": "daaedecd61ce4b9f0c77", "model": "Claude Opus 4.6 (120K, High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6917, "raw_score": 0.6917, "raw_scale": 1, "source_record": "claude-opus-4-6-thinking-120K-high", "cost_per_task": 3.4719 }, { "id": "e25abe981c2f8527f198", "model": "GPT-6 Sol (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.6888888888888888, "raw_score": 0.6888888888888888, "raw_scale": 1, "source_record": "openai-gpt-6-sol-high", "cost_per_task": 0.20273668333333333 }, { "id": "109cba51d4304208103a", "model": "Claude Opus 4.6 (120K, Max)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6875, "raw_score": 0.6875, "raw_scale": 1, "source_record": "claude-opus-4-6-thinking-120K-max", "cost_per_task": 3.641 }, { "id": "3e48bdfc9031606edd93", "model": "DeepSeek V4.1 Flash 1017 (High)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.6861111111111112, "raw_score": 0.6861111111111112, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-1017-high", "cost_per_task": 0.14693756500000005 }, { "id": "f98d8b6423dcaaf075d0", "model": "Claude 4.7 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6833, "raw_score": 0.6833, "raw_scale": 1, "source_record": "claude-4-7-high", "cost_per_task": 3.17 }, { "id": "791855d42afe07a67180", "model": "Claude 4.7 (Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 0.675, "raw_score": 0.675, "raw_scale": 1, "source_record": "claude-4-7-medium", "cost_per_task": 2.96 }, { "id": "8dd7bf216728fcb59495", "model": "DeepSeek V4.1 Flash (High)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.675, "raw_score": 0.675, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-high", "cost_per_task": 0.13341649249999996 }, { "id": "33915e78119e76090811", "model": "GPT-5.4 (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.675, "raw_score": 0.675, "raw_scale": 1, "source_record": "gpt-5-4-high", "cost_per_task": 1.02 }, { "id": "b37a5bcf247974b48296", "model": "Claude Sonnet 5.5 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6736111111111112, "raw_score": 0.6736111111111112, "raw_scale": 1, "source_record": "anthropic-claude-sonnet-5-5-high", "cost_per_task": 0.2767547333333332 }, { "id": "2e82481c7521128e4483", "model": "Grok 4.6 (XHigh)", "organization": "xAI", "configuration": "CoT", "score": 0.6708333333333333, "raw_score": 0.6708333333333333, "raw_scale": 1, "source_record": "xai-grok-4-6-xhigh", "cost_per_task": 0.7574904499999999 }, { "id": "567654a9097d37b75f66", "model": "GPT-5.6 Sol (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.6708, "raw_score": 0.6708, "raw_scale": 1, "source_record": "openai-gpt-5-6-sol-medium", "cost_per_task": 0.47 }, { "id": "94ec53d3941e9136a4b2", "model": "GPT-5.6 Terra (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.6708, "raw_score": 0.6708, "raw_scale": 1, "source_record": "openai-gpt-5-6-terra-high", "cost_per_task": 0.55 }, { "id": "b8ea20e911b8dabd0e3f", "model": "Claude Opus 4.6 (120K, Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6625, "raw_score": 0.6625, "raw_scale": 1, "source_record": "claude-opus-4-6-thinking-120K-medium", "cost_per_task": 2.7209 }, { "id": "906754bb72b2dd3686ac", "model": "GLM-5.3-Flash (Max)", "organization": "Z.ai", "configuration": "CoT", "score": 0.6583333333333333, "raw_score": 0.6583333333333333, "raw_scale": 1, "source_record": "zai-glm-5-3-flash-max", "cost_per_task": 0.09331568458333339 }, { "id": "5191c6c19246362dca25", "model": "Grok 4.20 (Reasoning)", "organization": "xAI", "configuration": "CoT", "score": 0.6514, "raw_score": 0.6514, "raw_scale": 1, "source_record": "grok-4.20-beta-0309b-reasoning", "cost_per_task": 0.92 }, { "id": "5f8134e2672e0a2e8078", "model": "Grok 4.6 (High)", "organization": "xAI", "configuration": "CoT", "score": 0.6513888888888888, "raw_score": 0.6513888888888888, "raw_scale": 1, "source_record": "xai-grok-4-6-high", "cost_per_task": 0.7338575666666665 }, { "id": "56cdbe4de086cd7dd179", "model": "Claude Opus 4.6 (120K, Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6458, "raw_score": 0.6458, "raw_scale": 1, "source_record": "claude-opus-4-6-thinking-120K-low", "cost_per_task": 2.2507 }, { "id": "ed7cfd450d9aae0f6c3d", "model": "Gemini 3.7 Flash (Medium)", "organization": "Google", "configuration": "CoT", "score": 0.6375, "raw_score": 0.6375, "raw_scale": 1, "source_record": "google-gemini-3-7-flash-medium", "cost_per_task": 0.11582842499999997 }, { "id": "389f98a99d1e98b23ce6", "model": "Claude Opus 4.8 (Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6222, "raw_score": 0.6222, "raw_scale": 1, "source_record": "anthropic-opus-4-8-low", "cost_per_task": 1.6818 }, { "id": "6ff9b2b91bb14a129d37", "model": "Claude 4.7 (Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6208, "raw_score": 0.6208, "raw_scale": 1, "source_record": "claude-4-7-low", "cost_per_task": 2.38 }, { "id": "a7333de805411cae110c", "model": "DeepSeek V4 Flash 0731 (Max)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.6138888888888888, "raw_score": 0.6138888888888888, "raw_scale": 1, "source_record": "deepseek-v4-flash-0731-max", "cost_per_task": 0.04163034416666667 }, { "id": "413faec415be6f532d49", "model": "Grok 4.7 (XHigh)", "organization": "xAI", "configuration": "CoT", "score": 0.6138888888888888, "raw_score": 0.6138888888888888, "raw_scale": 1, "source_record": "xai-grok-4-7-xhigh", "cost_per_task": 2.0149060666666663 }, { "id": "9b5093fdc2bd9d500c89", "model": "Claude Haiku 5.5 (XHigh)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6125, "raw_score": 0.6125, "raw_scale": 1, "source_record": "anthropic-claude-haiku-5-5-xhigh", "cost_per_task": 0.08498917333333329 }, { "id": "7c11b0920937558e965f", "model": "DeepSeek V4 Pro 0813 (Max)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.6125, "raw_score": 0.6125, "raw_scale": 1, "source_record": "deepseek-v4-pro-0813-max", "cost_per_task": 0.5976109150000003 }, { "id": "4aa41c705384198b535a", "model": "Grok 4.6 (Medium)", "organization": "xAI", "configuration": "CoT", "score": 0.6125, "raw_score": 0.6125, "raw_scale": 1, "source_record": "xai-grok-4-6-medium", "cost_per_task": 0.6586968666666668 }, { "id": "ead5de54124aeafd1da3", "model": "DeepSeek V4.1 Flash (Low)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.6055555555555555, "raw_score": 0.6055555555555555, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-low", "cost_per_task": 0.11904162500000005 }, { "id": "4cf5bba7e5037b674037", "model": "Claude Sonnet 4.6 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6042, "raw_score": 0.6042, "raw_scale": 1, "source_record": "claude_sonnet_4_6_high", "cost_per_task": 2.6966 }, { "id": "72ba8c06ab5e3681c5c6", "model": "Gemini 3.6 Flash (High)", "organization": "Google", "configuration": "CoT", "score": 0.6041666666666666, "raw_score": 0.6041666666666666, "raw_scale": 1, "source_record": "gemini-3-6-flash-high", "cost_per_task": 0.6060796625000002 }, { "id": "4846a085e53227051cc9", "model": "Kimi K3 (Max)", "organization": "Moonshot AI", "configuration": "CoT", "score": 0.6041666666666666, "raw_score": 0.6041666666666666, "raw_scale": 1, "source_record": "moonshot-kimi-k3-max", "cost_per_task": 1.5926940250000003 }, { "id": "1d47f4b3e786a2fa7691", "model": "DeepSeek V4 Pro 0813 (High)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.5972222222222221, "raw_score": 0.5972222222222221, "raw_scale": 1, "source_record": "deepseek-v4-pro-0813-high", "cost_per_task": 0.5969432040000002 }, { "id": "3bd14341f86cb32d704b", "model": "GPT-5.6 Luna 2026-07-30 (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5958333333333333, "raw_score": 0.5958333333333333, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-max-2026-07-30", "cost_per_task": 0.17671712499999995 }, { "id": "022d4147ea0203146774", "model": "GPT-6 Astra (None)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5958333333333333, "raw_score": 0.5958333333333333, "raw_scale": 1, "source_record": "openai-gpt-6-astra-none", "cost_per_task": 0.3696743333333333 }, { "id": "590a59ff94fd4a4cac2f", "model": "GPT-5.6 Luna (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5954, "raw_score": 0.5954, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-max", "cost_per_task": 0.67 }, { "id": "b541104d28447e2b9710", "model": "GPT-6 Luna (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5930555555555556, "raw_score": 0.5930555555555556, "raw_scale": 1, "source_record": "openai-gpt-6-luna-max", "cost_per_task": 0.06181100083333334 }, { "id": "54e3085d30ea73dc262d", "model": "Grok 4.7 (Medium)", "organization": "xAI", "configuration": "CoT", "score": 0.5875, "raw_score": 0.5875, "raw_scale": 1, "source_record": "xai-grok-4-7-medium", "cost_per_task": 1.4759098000000002 }, { "id": "1788f7609e36bad08f93", "model": "Grok 4.7 (High)", "organization": "xAI", "configuration": "CoT", "score": 0.5833333333333334, "raw_score": 0.5833333333333334, "raw_scale": 1, "source_record": "xai-grok-4-7-high", "cost_per_task": 1.549013099999999 }, { "id": "c0aa9c86d755f9b40ab8", "model": "Claude Sonnet 4.6 (Max)", "organization": "Anthropic", "configuration": "CoT", "score": 0.5833, "raw_score": 0.5833, "raw_scale": 1, "source_record": "claude_sonnet_4_6_max", "cost_per_task": 2.7179 }, { "id": "7f7f1f8e5c63703b9a99", "model": "GPT-6 Sol (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5777777777777778, "raw_score": 0.5777777777777778, "raw_scale": 1, "source_record": "openai-gpt-6-sol-medium", "cost_per_task": 0.14762708333333335 }, { "id": "9fde085da79f10d9b26f", "model": "Muse Spark Open (XHigh)", "organization": "Meta AI", "configuration": "CoT", "score": 0.575, "raw_score": 0.575, "raw_scale": 1, "source_record": "meta-muse-spark-open-xhigh", "cost_per_task": 0.7284857499999998 }, { "id": "bf5ba509452334a09240", "model": "DeepSeek V4 Pro 0813 (Low)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.5625, "raw_score": 0.5625, "raw_scale": 1, "source_record": "deepseek-v4-pro-0813-low", "cost_per_task": 0.37025881200000005 }, { "id": "4d0ec946871aedeb8a61", "model": "DeepSeek V4 Flash 0731 (High)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.5597222222222221, "raw_score": 0.5597222222222221, "raw_scale": 1, "source_record": "deepseek-v4-flash-0731-high", "cost_per_task": 0.0453381305 }, { "id": "5310fde93ef6d8dfe179", "model": "GPT-5.4 (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5542, "raw_score": 0.5542, "raw_scale": 1, "source_record": "gpt-5-4-medium", "cost_per_task": 0.68 }, { "id": "b64c32cfd59681297d20", "model": "Kimi K3 (High)", "organization": "Moonshot AI", "configuration": "CoT", "score": 0.55, "raw_score": 0.55, "raw_scale": 1, "source_record": "moonshot-kimi-k3-high", "cost_per_task": 0.9472229249999998 }, { "id": "b0751ad41fadbb5ce0e1", "model": "GPT-5.2 Pro (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5416, "raw_score": 0.5416, "raw_scale": 1, "source_record": "gpt-5-2-pro-2025-12-11-high", "cost_per_task": 15.721 }, { "id": "f7f8ea934abefefeeb8a", "model": "Gemini 3 Pro (Refine.)", "organization": "Google", "configuration": "Refinement; Poetiq", "score": 0.54, "raw_score": 0.54, "raw_scale": 1, "source_record": "poetiq-arc-agi-gemini-3-d_v1", "cost_per_task": 30.568 }, { "id": "fdb4c68b4d13ab04d5d4", "model": "Gemini 3.7 Flash (Low)", "organization": "Google", "configuration": "CoT", "score": 0.5291666666666667, "raw_score": 0.5291666666666667, "raw_scale": 1, "source_record": "google-gemini-3-7-flash-low", "cost_per_task": 0.07938655 }, { "id": "705c23fafccdc8f8aee3", "model": "GPT-5.2 (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5291, "raw_score": 0.5291, "raw_scale": 1, "source_record": "gpt-5-2-2025-12-11-thinking-xhigh", "cost_per_task": 1.8982 }, { "id": "16f0d560909bc42dba3d", "model": "Grok 4.5 (High)", "organization": "xAI", "configuration": "CoT", "score": 0.5264, "raw_score": 0.5264, "raw_scale": 1, "source_record": "xai-grok-4-5-high", "cost_per_task": 0.78 }, { "id": "8c1563d40d75c4056d3c", "model": "Grok 4.5 (Medium)", "organization": "xAI", "configuration": "CoT", "score": 0.5264, "raw_score": 0.5264, "raw_scale": 1, "source_record": "xai-grok-4-5-medium", "cost_per_task": 0.75 }, { "id": "c888dd009751e5138837", "model": "Gemini 3.6 Flash (Medium)", "organization": "Google", "configuration": "CoT", "score": 0.5041666666666667, "raw_score": 0.5041666666666667, "raw_scale": 1, "source_record": "gemini-3-6-flash-medium", "cost_per_task": 0.45550909999999994 }, { "id": "03128c2eeca1a9610c03", "model": "GLM-5.3-Flash (High)", "organization": "Z.ai", "configuration": "CoT", "score": 0.5013888888888889, "raw_score": 0.5013888888888889, "raw_scale": 1, "source_record": "zai-glm-5-3-flash-high", "cost_per_task": 0.07734862875 }, { "id": "321ee56ecae91df6ca90", "model": "GPT-5.6 Luna (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.4764, "raw_score": 0.4764, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-xhigh", "cost_per_task": 0.54 }, { "id": "ad4009292d995e786f5a", "model": "Muse Spark Open (High)", "organization": "Meta AI", "configuration": "CoT", "score": 0.475, "raw_score": 0.475, "raw_scale": 1, "source_record": "meta-muse-spark-open-high", "cost_per_task": 0.6244857354166667 }, { "id": "b83c2ef567a2ad6c9833", "model": "DeepSeek V4 Flash 0731 (Low)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.4597222222222222, "raw_score": 0.4597222222222222, "raw_scale": 1, "source_record": "deepseek-v4-flash-0731-low", "cost_per_task": 0.0206284715 }, { "id": "5f075f011079ca197000", "model": "Gemini 3 Deep Think (Preview) ²", "organization": "Google", "configuration": "CoT", "score": 0.4514, "raw_score": 0.4514, "raw_scale": 1, "source_record": "gemini-3-deep-think-preview", "cost_per_task": 77.16309638 }, { "id": "4243156c85c87a6efe19", "model": "GPT-5.6 Luna 2026-07-30 (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.4347222222222222, "raw_score": 0.4347222222222222, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-xhigh-2026-07-30", "cost_per_task": 0.13048743000000002 }, { "id": "a85a209e494b39c4b690", "model": "GPT-5.2 (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.4333, "raw_score": 0.4333, "raw_scale": 1, "source_record": "gpt-5-2-2025-12-11-thinking-high", "cost_per_task": 1.393 }, { "id": "fa4719b3453045f8dca6", "model": "GPT-5.6 Sol (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.425, "raw_score": 0.425, "raw_scale": 1, "source_record": "openai-gpt-5-6-sol-low", "cost_per_task": 0.32 }, { "id": "c56bba87bdf1801196ff", "model": "Qwen3.8-27B (XHigh)", "organization": "Alibaba", "configuration": "CoT", "score": 0.4236111111111111, "raw_score": 0.4236111111111111, "raw_scale": 1, "source_record": "alibaba-qwen3-8-27b-xhigh", "cost_per_task": 0.4469396510416667 }, { "id": "92bc321cbf49e327bf2e", "model": "Gemini 4 Argon (Low)", "organization": "Google", "configuration": "CoT", "score": 0.42083333333333334, "raw_score": 0.42083333333333334, "raw_scale": 1, "source_record": "google-gemini-4-argon-low", "cost_per_task": 0.07381080000000001 }, { "id": "6d278812004c63d9e241", "model": "GPT-6 Luna (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.41944444444444445, "raw_score": 0.41944444444444445, "raw_scale": 1, "source_record": "openai-gpt-6-luna-xhigh", "cost_per_task": 0.032297893333333334 }, { "id": "40d693c110db2eb4de23", "model": "Inkling Small (XHigh)", "organization": "Thinking Machines", "configuration": "CoT", "score": 0.40138888888888885, "raw_score": 0.40138888888888885, "raw_scale": 1, "source_record": "thinky-inkling-small-xhigh", "cost_per_task": 0.23255201249999993 }, { "id": "c8c1043888fb73a75edf", "model": "GPT-5.2 Pro (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.3847, "raw_score": 0.3847, "raw_scale": 1, "source_record": "gpt-5-2-pro-2025-12-11-medium", "cost_per_task": 8.9928 }, { "id": "65ca6b9c2f1909594725", "model": "Opus 4.5 (Thinking, 64K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.3764, "raw_score": 0.3764, "raw_scale": 1, "source_record": "claude-opus-4-5-20251101-thinking-64k", "cost_per_task": 2.4 }, { "id": "db537cb3526a5dcd3be8", "model": "GPT-5.6 Terra (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.375, "raw_score": 0.375, "raw_scale": 1, "source_record": "openai-gpt-5-6-terra-medium", "cost_per_task": 0.27 }, { "id": "65f56842e6429f23bb54", "model": "Inkling", "organization": "Thinking Machines", "configuration": "CoT", "score": 0.36527777777777776, "raw_score": 0.36527777777777776, "raw_scale": 1, "source_record": "thinky-inkling", "cost_per_task": 0.6420766277310923 }, { "id": "6d10a126137c335ed08f", "model": "Gemini 3 Flash Preview (High)", "organization": "Google", "configuration": "CoT", "score": 0.3361, "raw_score": 0.3361, "raw_scale": 1, "source_record": "gemini-3-flash-preview-thinking-high", "cost_per_task": 0.2314 }, { "id": "320f9af2e43d49dddcd4", "model": "GPT-5.6 Luna 2026-07-30 (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.3333333333333333, "raw_score": 0.3333333333333333, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-high-2026-07-30", "cost_per_task": 0.06253926000000003 }, { "id": "e7d73f6f686694fba3ce", "model": "GPT-5.5 (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.3333, "raw_score": 0.3333, "raw_scale": 1, "source_record": "gpt-5-5-2026-04-22-thinking-low", "cost_per_task": 0.35 }, { "id": "c061e2168a85bbdd3822", "model": "Grok 4.5 (Low)", "organization": "xAI", "configuration": "CoT", "score": 0.3306, "raw_score": 0.3306, "raw_scale": 1, "source_record": "xai-grok-4-5-low", "cost_per_task": 0.4 }, { "id": "6375864641df0da8ba11", "model": "Inkling Small (High)", "organization": "Thinking Machines", "configuration": "CoT", "score": 0.33055555555555555, "raw_score": 0.33055555555555555, "raw_scale": 1, "source_record": "thinky-inkling-small-high", "cost_per_task": 0.132909555 }, { "id": "b8fe27653f4ab0bbf881", "model": "Claude Sonnet 5.5 (Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 0.3277777777777777, "raw_score": 0.3277777777777777, "raw_scale": 1, "source_record": "anthropic-claude-sonnet-5-5-medium", "cost_per_task": 0.15537473333333335 }, { "id": "3524b0aa16e87d7e6de0", "model": "GPT-6 Sol (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.3152777777777777, "raw_score": 0.3152777777777777, "raw_scale": 1, "source_record": "openai-gpt-6-sol-low", "cost_per_task": 0.1008582166666667 }, { "id": "1043e5abc10417d0a6c6", "model": "GPT-6 Luna (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.3138888888888889, "raw_score": 0.3138888888888889, "raw_scale": 1, "source_record": "openai-gpt-6-luna-high", "cost_per_task": 0.02336191 }, { "id": "23d48bc5e67b8d100401", "model": "Gemini 3 Pro", "organization": "Google", "configuration": "CoT", "score": 0.3111, "raw_score": 0.3111, "raw_scale": 1, "source_record": "gemini-3-pro-preview", "cost_per_task": 0.8109 }, { "id": "29f4959c11090a2ca874", "model": "Claude Haiku 5.5 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.3069444444444444, "raw_score": 0.3069444444444444, "raw_scale": 1, "source_record": "anthropic-claude-haiku-5-5-high", "cost_per_task": 0.046677990833333315 }, { "id": "068c1cb11e9f490bd5ea", "model": "Gemini 3.6 Flash (Low)", "organization": "Google", "configuration": "CoT", "score": 0.30416666666666664, "raw_score": 0.30416666666666664, "raw_scale": 1, "source_record": "gemini-3-6-flash-low", "cost_per_task": 0.22297816250000002 }, { "id": "2b5246e67a9cd59b47b9", "model": "Grok 4 (Refine.)", "organization": "xAI", "configuration": "Refinement; J. Berman", "score": 0.2944, "raw_score": 0.2944, "raw_scale": 1, "source_record": "jeremy_sept_2025", "cost_per_task": 30.4 }, { "id": "21f6c06f670fb4ad76cd", "model": "GPT-5.6 Luna (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.2931, "raw_score": 0.2931, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-high", "cost_per_task": 0.29 }, { "id": "b505c555f70a2576d3a3", "model": "GPT-5.4 (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.2917, "raw_score": 0.2917, "raw_scale": 1, "source_record": "gpt-5-4-low", "cost_per_task": 0.27 }, { "id": "4e28f918a5d530650b0d", "model": "GLM-5.3-Flash (Low)", "organization": "Z.ai", "configuration": "CoT", "score": 0.2791666666666667, "raw_score": 0.2791666666666667, "raw_scale": 1, "source_record": "zai-glm-5-3-flash-low", "cost_per_task": 0.05822348833333333 }, { "id": "13e791c3e2cd867c08fd", "model": "Grok 4.6 (Low)", "organization": "xAI", "configuration": "CoT", "score": 0.2763888888888889, "raw_score": 0.2763888888888889, "raw_scale": 1, "source_record": "xai-grok-4-6-low", "cost_per_task": 0.2124022166666667 }, { "id": "61ea0169cfa800e75ef0", "model": "Muse Spark Open (Medium)", "organization": "Meta AI", "configuration": "CoT", "score": 0.275, "raw_score": 0.275, "raw_scale": 1, "source_record": "meta-muse-spark-open-medium", "cost_per_task": 0.2514419854166666 }, { "id": "33d5a86bdc8b7f823f92", "model": "GPT-5.2 (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.2667, "raw_score": 0.2667, "raw_scale": 1, "source_record": "gpt-5-2-2025-12-11-thinking-medium", "cost_per_task": 0.7594 }, { "id": "3b06ac765945063c23de", "model": "Claude Sonnet 5.5 (Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.24027777777777776, "raw_score": 0.24027777777777776, "raw_scale": 1, "source_record": "anthropic-claude-sonnet-5-5-low", "cost_per_task": 0.10923548333333331 }, { "id": "c43dcd330813e4f2479f", "model": "Beam 1.0 (XHigh)", "organization": "Reflection", "configuration": "CoT", "score": 0.23472222222222222, "raw_score": 0.23472222222222222, "raw_scale": 1, "source_record": "reflection-beam-1-0-xhigh", "cost_per_task": 0.6095804058333331 }, { "id": "7fabbf552115c4ca5562", "model": "Opus 4.5 (Thinking, 16K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.2278, "raw_score": 0.2278, "raw_scale": 1, "source_record": "claude-opus-4-5-20251101-thinking-16k", "cost_per_task": 0.79 }, { "id": "96a4d20a34adeb014f90", "model": "Qwen3.8-27B (Low)", "organization": "Alibaba", "configuration": "CoT", "score": 0.22777777777777777, "raw_score": 0.22777777777777777, "raw_scale": 1, "source_record": "alibaba-qwen3-8-27b-low", "cost_per_task": 0.2380080077083332 }, { "id": "bb43a640f04c8e5504a8", "model": "GLM-5.2", "organization": "Z.ai", "configuration": "CoT", "score": 0.22775, "raw_score": 0.22775, "raw_scale": 1, "source_record": "glm-5.2", "cost_per_task": 0.2501 }, { "id": "ee13de24fbf77c171565", "model": "Beam 1.0 (High)", "organization": "Reflection", "configuration": "CoT", "score": 0.2, "raw_score": 0.2, "raw_scale": 1, "source_record": "reflection-beam-1-0-high", "cost_per_task": 0.34779271500000003 }, { "id": "4664cd24464c799f4f4a", "model": "GPT-5.4 Mini (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.189, "raw_score": 0.189, "raw_scale": 1, "source_record": "gpt-5-4-mini-xhigh", "cost_per_task": 0.75 }, { "id": "82431eb5d0ed95b9cb4c", "model": "GPT-5.6 Terra (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.1875, "raw_score": 0.1875, "raw_scale": 1, "source_record": "openai-gpt-5-6-terra-low", "cost_per_task": 0.17 }, { "id": "5dab13fb1633331fb599", "model": "GPT-5 Pro", "organization": "OpenAI", "configuration": "CoT", "score": 0.1833, "raw_score": 0.1833, "raw_scale": 1, "source_record": "gpt-5-pro-2025-10-06", "cost_per_task": 7.1432 }, { "id": "e40f85b75d4b49d59614", "model": "GPT-6 Luna (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.18055555555555555, "raw_score": 0.18055555555555555, "raw_scale": 1, "source_record": "openai-gpt-6-luna-medium", "cost_per_task": 0.012202668333333333 }, { "id": "d0d73ea44dc36189a8f9", "model": "GPT-5.1 (Thinking, High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.1764, "raw_score": 0.1764, "raw_scale": 1, "source_record": "gpt-5-1-2025-11-13-thinking-high", "cost_per_task": 1.1704 }, { "id": "2fad3855b5e6e28f604a", "model": "Grok 4.7 (Low)", "organization": "xAI", "configuration": "CoT", "score": 0.16666666666666666, "raw_score": 0.16666666666666666, "raw_scale": 1, "source_record": "xai-grok-4-7-low", "cost_per_task": 0.17369760000000004 }, { "id": "307581d13620471e0613", "model": "Grok 4 (Thinking)", "organization": "xAI", "configuration": "CoT", "score": 0.15975, "raw_score": 0.15975, "raw_scale": 1, "source_record": "grok-4-0709", "cost_per_task": 2.1659 }, { "id": "fba5e5fb4a1f92d900de", "model": "Opus 4.5 (Thinking, 8K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.1389, "raw_score": 0.1389, "raw_scale": 1, "source_record": "claude-opus-4-5-20251101-thinking-8k", "cost_per_task": 0.48 }, { "id": "e8c97f8011b4019c28a7", "model": "Inkling Small (Medium)", "organization": "Thinking Machines", "configuration": "CoT", "score": 0.13611111111111113, "raw_score": 0.13611111111111113, "raw_scale": 1, "source_record": "thinky-inkling-small-medium", "cost_per_task": 0.06378986500000002 }, { "id": "bf7b9402ef8ccc7fee15", "model": "Claude Sonnet 4.5 (Thinking 32K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.1361, "raw_score": 0.1361, "raw_scale": 1, "source_record": "claude-sonnet-4-5-20250929-thinking-32k", "cost_per_task": 0.7589 }, { "id": "565f2944e58169719d97", "model": "Qwen3.8-27B (Medium)", "organization": "Alibaba", "configuration": "CoT", "score": 0.13194444444444445, "raw_score": 0.13194444444444445, "raw_scale": 1, "source_record": "alibaba-qwen3-8-27b-medium", "cost_per_task": 0.2862669347916667 }, { "id": "bfc90657677ac925afb8", "model": "GPT-5.4 Mini (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.1319, "raw_score": 0.1319, "raw_scale": 1, "source_record": "gpt-5-4-mini-high", "cost_per_task": 0.56 }, { "id": "fbc5a0c0e18b40a2ef2c", "model": "Gemini 3 Flash Preview (Medium)", "organization": "Google", "configuration": "CoT", "score": 0.1278, "raw_score": 0.1278, "raw_scale": 1, "source_record": "gemini-3-flash-preview-thinking-medium", "cost_per_task": 0.0824 }, { "id": "d8d55724101f81643da1", "model": "Kimi K3 (Low)", "organization": "Moonshot AI", "configuration": "CoT", "score": 0.12361111111111112, "raw_score": 0.12361111111111112, "raw_scale": 1, "source_record": "moonshot-kimi-k3-low", "cost_per_task": 0.2541386750000001 }, { "id": "a2c5f75336f599ce1752", "model": "Muse Spark Open (Low)", "organization": "Meta AI", "configuration": "CoT", "score": 0.12222222222222222, "raw_score": 0.12222222222222222, "raw_scale": 1, "source_record": "meta-muse-spark-open-low", "cost_per_task": 0.10650972499999997 }, { "id": "47ab0eb3862b80afd346", "model": "Kimi K2.5", "organization": "Moonshot AI", "configuration": "CoT", "score": 0.1181, "raw_score": 0.1181, "raw_scale": 1, "source_record": "kimi-k2.5", "cost_per_task": 0.28 }, { "id": "970d3a88a140e9d16e9e", "model": "Gemini 3.5 Flash-Lite (High)", "organization": "Google", "configuration": "CoT", "score": 0.10277777777777779, "raw_score": 0.10277777777777779, "raw_scale": 1, "source_record": "gemini-3-5-flash-lite-high", "cost_per_task": 0.13899638250000004 }, { "id": "8b597a8a2ef7523fca68", "model": "GPT-5 (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0986, "raw_score": 0.0986, "raw_scale": 1, "source_record": "gpt-5-2025-08-07-high", "cost_per_task": 0.7302 }, { "id": "612b2d44d2d0fa33f7aa", "model": "GPT-5.2 (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0972, "raw_score": 0.0972, "raw_scale": 1, "source_record": "gpt-5-2-2025-12-11-thinking-low", "cost_per_task": 0.2642 }, { "id": "6a439682a4bced3ea109", "model": "Claude Haiku 5.5 (Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 0.09444444444444443, "raw_score": 0.09444444444444443, "raw_scale": 1, "source_record": "anthropic-claude-haiku-5-5-medium", "cost_per_task": 0.014974957499999999 }, { "id": "d1b487a9f3369cd962a4", "model": "Gemini 3.5 Flash (Minimal)", "organization": "Google", "configuration": "CoT", "score": 0.0889, "raw_score": 0.0889, "raw_scale": 1, "source_record": "gemini-3-5-flash-minimal", "cost_per_task": 0.1074 }, { "id": "35413170b9929d46621f", "model": "Claude Opus 4 (Thinking 16K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0861, "raw_score": 0.0861, "raw_scale": 1, "source_record": "claude-opus-4-20250514-thinking-16k", "cost_per_task": 1.9284 }, { "id": "6695f13d0c78c15a0e4c", "model": "Opus 4.5 (Thinking, None)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0778, "raw_score": 0.0778, "raw_scale": 1, "source_record": "claude-opus-4-5-20251101-thinking-none", "cost_per_task": 0.22 }, { "id": "be74e0ac7719a919707c", "model": "GPT-5 (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0749, "raw_score": 0.0749, "raw_scale": 1, "source_record": "gpt-5-2025-08-07-medium", "cost_per_task": 0.4486 }, { "id": "50a86958f5ae9d945ab6", "model": "GPT-5.6 Luna 2026-07-30 (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0736111111111111, "raw_score": 0.0736111111111111, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-medium-2026-07-30", "cost_per_task": 0.02209704 }, { "id": "203f3e30ee8c4f8f6b02", "model": "GPT-5.6 Luna (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0736, "raw_score": 0.0736, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-medium", "cost_per_task": 0.11 }, { "id": "d716739860bd3013e08d", "model": "Claude Sonnet 4.5 (Thinking 16K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0694, "raw_score": 0.0694, "raw_scale": 1, "source_record": "claude-sonnet-4-5-20250929-thinking-16k", "cost_per_task": 0.3499 }, { "id": "1ee0d27c96f21091b850", "model": "Claude Sonnet 4.5 (Thinking 8K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0694, "raw_score": 0.0694, "raw_scale": 1, "source_record": "claude-sonnet-4-5-20250929-thinking-8k", "cost_per_task": 0.2349 }, { "id": "8d422f8b3c919c08ed40", "model": "GPT-5.1 (Thinking, Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0653, "raw_score": 0.0653, "raw_scale": 1, "source_record": "gpt-5-1-2025-11-13-thinking-medium", "cost_per_task": 0.4206 }, { "id": "6578fedbd4adf9ca25be", "model": "o3 (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0653, "raw_score": 0.0653, "raw_scale": 1, "source_record": "o3-2025-04-16-high", "cost_per_task": 0.8339 }, { "id": "e0af5c9ebe19e93a78f0", "model": "Beam 1.0 (Medium)", "organization": "Reflection", "configuration": "CoT", "score": 0.0638888888888889, "raw_score": 0.0638888888888889, "raw_scale": 1, "source_record": "reflection-beam-1-0-medium", "cost_per_task": 0.18042641083333338 }, { "id": "b16db04215ea6629cfee", "model": "o4-mini (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0611, "raw_score": 0.0611, "raw_scale": 1, "source_record": "o4-mini-2025-04-16-high", "cost_per_task": 0.856 }, { "id": "337e138f10ed252ae211", "model": "Claude Sonnet 4 (Thinking 16K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0593, "raw_score": 0.0593, "raw_scale": 1, "source_record": "claude-sonnet-4-20250514-thinking-16k-bedrock", "cost_per_task": 0.4857 }, { "id": "6f71b70f2383e090a81b", "model": "Claude Sonnet 4.5 (Thinking 1K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0583, "raw_score": 0.0583, "raw_scale": 1, "source_record": "claude-sonnet-4-5-20250929-thinking-1k", "cost_per_task": 0.1424 }, { "id": "8531a9139a5d02e5954b", "model": "GPT-5.4 Nano (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0569, "raw_score": 0.0569, "raw_scale": 1, "source_record": "gpt-5-4-nano-xhigh", "cost_per_task": 0.16 }, { "id": "1935a8da3e6c8825c196", "model": "Grok 4 (Fast Reasoning)", "organization": "xAI", "configuration": "CoT", "score": 0.0528, "raw_score": 0.0528, "raw_scale": 1, "source_record": "grok-4-fast-reasoning", "cost_per_task": 0.061 }, { "id": "f0ed41566b869219d9ae", "model": "Gemini 3.5 Flash-Lite (Medium)", "organization": "Google", "configuration": "CoT", "score": 0.05277777777777778, "raw_score": 0.05277777777777778, "raw_scale": 1, "source_record": "gemini-3-5-flash-lite-medium", "cost_per_task": 0.04367807000000001 }, { "id": "7ed33f9a02fa1868094f", "model": "GPT-5.6 Luna (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0514, "raw_score": 0.0514, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-low", "cost_per_task": 0.07 }, { "id": "f83bfc4f1229d1f65d92", "model": "Inkling Small (Low)", "organization": "Thinking Machines", "configuration": "CoT", "score": 0.04861111111111112, "raw_score": 0.04861111111111112, "raw_scale": 1, "source_record": "thinky-inkling-small-low", "cost_per_task": 0.05589815499999999 }, { "id": "343fb7bd0d85299db470", "model": "GLM-5", "organization": "Z.ai", "configuration": "CoT", "score": 0.0486, "raw_score": 0.0486, "raw_scale": 1, "source_record": "glm-5", "cost_per_task": 0.27 }, { "id": "040c29ded3334582d003", "model": "Gemini 2.5 Pro (Thinking 32K)", "organization": "Google", "configuration": "CoT", "score": 0.0486, "raw_score": 0.0486, "raw_scale": 1, "source_record": "gemini-2-5-pro-2025-06-17-thinking-32k", "cost_per_task": 0.757 }, { "id": "835996680aaa0ca166b4", "model": "Minimax M2.5", "organization": "Minimax", "configuration": "CoT", "score": 0.0486, "raw_score": 0.0486, "raw_scale": 1, "source_record": "minimax-m2.5", "cost_per_task": 0.17 }, { "id": "fa22c420d0f0c695e2a6", "model": "o3-Pro (High)", "organization": "OpenAI", "configuration": "CoT + Synthesis", "score": 0.0486, "raw_score": 0.0486, "raw_scale": 1, "source_record": "o3-pro-2025-06-10-high", "cost_per_task": 7.5516 }, { "id": "25389e39b01c73d015ca", "model": "GPT-6 Luna (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.04583333333333333, "raw_score": 0.04583333333333333, "raw_scale": 1, "source_record": "openai-gpt-6-luna-low", "cost_per_task": 0.005273818333333334 }, { "id": "b9d620595374a6b72325", "model": "Claude Opus 4 (Thinking 8K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0452, "raw_score": 0.0452, "raw_scale": 1, "source_record": "claude-opus-4-20250514-thinking-8k", "cost_per_task": 1.1569 }, { "id": "2338f40120ec4075e7e3", "model": "GPT-5 Mini (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0444, "raw_score": 0.0444, "raw_scale": 1, "source_record": "gpt-5-mini-2025-08-07-high", "cost_per_task": 0.1977 }, { "id": "22c39689b4a43a1a3e40", "model": "GPT-5.4 Mini (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0444, "raw_score": 0.0444, "raw_scale": 1, "source_record": "gpt-5-4-mini-medium", "cost_per_task": 0.33 }, { "id": "330a4df31a98ae1122e1", "model": "Claude Haiku 4.5 (Thinking 32K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0403, "raw_score": 0.0403, "raw_scale": 1, "source_record": "claude-haiku-4-5-20251001-thinking-32k", "cost_per_task": 0.3766 }, { "id": "c4e6a3d9e5d22f4b120b", "model": "Deepseek V3.2", "organization": "DeepSeek", "configuration": "Base LLM", "score": 0.0403, "raw_score": 0.0403, "raw_scale": 1, "source_record": "deepseek-v3.2", "cost_per_task": 0.12 }, { "id": "6049fbb155d514069d06", "model": "GPT-5 Mini (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0403, "raw_score": 0.0403, "raw_scale": 1, "source_record": "gpt-5-mini-2025-08-07-medium", "cost_per_task": 0.0629 }, { "id": "30f4e4391e5717b7ce86", "model": "Gemini 2.5 Pro (Thinking 16K)", "organization": "Google", "configuration": "CoT", "score": 0.0403, "raw_score": 0.0403, "raw_scale": 1, "source_record": "gemini-2-5-pro-2025-06-17-thinking-16k", "cost_per_task": 0.7145 }, { "id": "dd6eeccef2c6d81b9730", "model": "Inkling Small (Minimal)", "organization": "Thinking Machines", "configuration": "CoT", "score": 0.04027777777777777, "raw_score": 0.04027777777777777, "raw_scale": 1, "source_record": "thinky-inkling-small-minimal", "cost_per_task": 0.05303395499999999 }, { "id": "efcf30858a13f54daaca", "model": "Claude Sonnet 4.5", "organization": "Anthropic", "configuration": "Base LLM", "score": 0.0375, "raw_score": 0.0375, "raw_scale": 1, "source_record": "claude-sonnet-4-5-20250929", "cost_per_task": 0.1295 }, { "id": "82d51f1ec1133c9e9f81", "model": "GPT-5.6 Luna 2026-07-30 (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.03611111111111111, "raw_score": 0.03611111111111111, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-low-2026-07-30", "cost_per_task": 0.013629859999999995 }, { "id": "167808d0752b8c13fe07", "model": "GPT-5.4 Nano (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0361, "raw_score": 0.0361, "raw_scale": 1, "source_record": "gpt-5-4-nano-high", "cost_per_task": 0.13 }, { "id": "54e3a0ce5a3318d05def", "model": "Gemini 3 Flash Preview (Minimal)", "organization": "Google", "configuration": "CoT", "score": 0.0333, "raw_score": 0.0333, "raw_scale": 1, "source_record": "gemini-3-flash-preview-thinking-minimal", "cost_per_task": 0.0205 }, { "id": "0118c3f427af85b45ecf", "model": "o3-mini (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0299, "raw_score": 0.0299, "raw_scale": 1, "source_record": "o3-mini-2025-01-31-high", "cost_per_task": 0.5472 }, { "id": "02c7f1241551d2f6321c", "model": "o3 (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0298, "raw_score": 0.0298, "raw_scale": 1, "source_record": "o3-2025-04-16-medium", "cost_per_task": 0.4787 }, { "id": "4656b3832eb4263991b6", "model": "Gemini 2.5 Pro (Thinking 8K)", "organization": "Google", "configuration": "CoT", "score": 0.0292, "raw_score": 0.0292, "raw_scale": 1, "source_record": "gemini-2-5-pro-2025-06-17-thinking-8k", "cost_per_task": 0.4439 }, { "id": "8368473413618c4395c0", "model": "Claude Haiku 4.5 (Thinking 16K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0278, "raw_score": 0.0278, "raw_scale": 1, "source_record": "claude-haiku-4-5-20251001-thinking-16k", "cost_per_task": 0.1392 }, { "id": "4aa72d137dea418631e3", "model": "Beam 1.0 (Low)", "organization": "Reflection", "configuration": "CoT", "score": 0.02777777777777778, "raw_score": 0.02777777777777778, "raw_scale": 1, "source_record": "reflection-beam-1-0-low", "cost_per_task": 0.10010559999999999 }, { "id": "b2aecc7fe18b3129bd41", "model": "Claude Haiku 5.5 (Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.027777777777777776, "raw_score": 0.027777777777777776, "raw_scale": 1, "source_record": "anthropic-claude-haiku-5-5-low", "cost_per_task": 0.007986432500000003 }, { "id": "88c4251b170bebbdfd16", "model": "Gemini 3.6 Flash (Minimal)", "organization": "Google", "configuration": "CoT", "score": 0.02638888888888889, "raw_score": 0.02638888888888889, "raw_scale": 1, "source_record": "gemini-3-6-flash-minimal", "cost_per_task": 0.07084053750000002 }, { "id": "286f99850e8e5918fa01", "model": "GPT-5 Nano (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0261, "raw_score": 0.0261, "raw_scale": 1, "source_record": "gpt-5-nano-2025-08-07-high", "cost_per_task": 0.0295 }, { "id": "cd009c68618c6a2ed163", "model": "Gemini 2.5 Flash (Preview) (Thinking 24K)", "organization": "Google", "configuration": "CoT", "score": 0.0254, "raw_score": 0.0254, "raw_scale": 1, "source_record": "gemini-2-5-flash-preview-05-20-thinking-24k", "cost_per_task": 0.3191 }, { "id": "09b2fc78b17bfbaec2a9", "model": "o4-mini (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0236, "raw_score": 0.0236, "raw_scale": 1, "source_record": "o4-mini-2025-04-16-medium", "cost_per_task": 0.2311 }, { "id": "a2a8cbfd99b8eb2a99a2", "model": "Gemini 2.5 Flash (Preview) (Thinking 1K)", "organization": "Google", "configuration": "CoT", "score": 0.0216, "raw_score": 0.0216, "raw_scale": 1, "source_record": "gemini-2-5-flash-preview-05-20-thinking-1k", "cost_per_task": 0.0302 }, { "id": "b8cb163ea989c4ca0e2e", "model": "Claude Sonnet 4 (Thinking 8K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0212, "raw_score": 0.0212, "raw_scale": 1, "source_record": "claude-sonnet-4-20250514-thinking-8k-bedrock", "cost_per_task": 0.2654 }, { "id": "59d35b606f9d2f2faeac", "model": "Gemini 2.5 Flash (Preview) (Thinking 8K)", "organization": "Google", "configuration": "CoT", "score": 0.0212, "raw_score": 0.0212, "raw_scale": 1, "source_record": "gemini-2-5-flash-preview-05-20-thinking-8k", "cost_per_task": 0.1994 }, { "id": "38bc95c4877821a67c0d", "model": "DeepSeek V4 Flash 0731 (None)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.020833333333333332, "raw_score": 0.020833333333333332, "raw_scale": 1, "source_record": "deepseek-v4-flash-0731-none", "cost_per_task": 0.0037609368333333336 }, { "id": "c0d4177931a986925128", "model": "DeepSeek V4.1 Flash 1017 (None)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.020833333333333332, "raw_score": 0.020833333333333332, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-1017-none", "cost_per_task": 0.010167875000000003 }, { "id": "90a7984fe355877999ec", "model": "o3-mini (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0208, "raw_score": 0.0208, "raw_scale": 1, "source_record": "o3-mini-2025-01-31-medium", "cost_per_task": 0.2843 }, { "id": "d1d01e163fa0e19c4c1a", "model": "o3-Pro (Low)", "organization": "OpenAI", "configuration": "CoT + Synthesis", "score": 0.0205, "raw_score": 0.0205, "raw_scale": 1, "source_record": "o3-pro-2025-06-10-low", "cost_per_task": 2.2293 }, { "id": "364957e2ce19b8cdf67f", "model": "o3 (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0199, "raw_score": 0.0199, "raw_scale": 1, "source_record": "o3-2025-04-16-low", "cost_per_task": 0.2343 }, { "id": "bf3809b8d3365e6faeea", "model": "Gemini 2.5 Flash (Preview) (Thinking 16K)", "organization": "Google", "configuration": "CoT", "score": 0.0198, "raw_score": 0.0198, "raw_scale": 1, "source_record": "gemini-2-5-flash-preview-05-20-thinking-16k", "cost_per_task": 0.3173 }, { "id": "dbd5098a143294ddc06b", "model": "Nemotron 3.5 Super VL (Medium)", "organization": "NVIDIA", "configuration": "CoT", "score": 0.019444444444444445, "raw_score": 0.019444444444444445, "raw_scale": 1, "source_record": "nvidia-nemotron-3-5-super-vl-120b-a12b-medium", "cost_per_task": 0.017309603333333326 }, { "id": "16c78eb464857542d2f0", "model": "GPT-5 (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0194, "raw_score": 0.0194, "raw_scale": 1, "source_record": "gpt-5-2025-08-07-low", "cost_per_task": 0.1896 }, { "id": "5a5d58abe4575f7bd64c", "model": "GPT-5.1 (Thinking, Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0194, "raw_score": 0.0194, "raw_scale": 1, "source_record": "gpt-5-1-2025-11-13-thinking-low", "cost_per_task": 0.129 }, { "id": "4a099160053751b91643", "model": "GPT-5.4 Nano (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0194, "raw_score": 0.0194, "raw_scale": 1, "source_record": "gpt-5-4-nano-medium", "cost_per_task": 0.06 }, { "id": "322baeb812128a21adc0", "model": "o3-Pro (Medium)", "organization": "OpenAI", "configuration": "CoT + Synthesis", "score": 0.0194, "raw_score": 0.0194, "raw_scale": 1, "source_record": "o3-pro-2025-06-10-medium", "cost_per_task": 4.7441 }, { "id": "51f56f40b1ff34a7de82", "model": "Gemini 2.5 Flash (Preview)", "organization": "Google", "configuration": "CoT", "score": 0.0169, "raw_score": 0.0169, "raw_scale": 1, "source_record": "gemini-2-5-flash-preview-05-20", "cost_per_task": 0.057 }, { "id": "c0da1ab3126e75090bdc", "model": "Claude Haiku 4.5 (Thinking 8K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0167, "raw_score": 0.0167, "raw_scale": 1, "source_record": "claude-haiku-4-5-20251001-thinking-8k", "cost_per_task": 0.0909 }, { "id": "a3a3a03e898e82b603ed", "model": "GPT-5 Mini (Minimal)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0167, "raw_score": 0.0167, "raw_scale": 1, "source_record": "gpt-5-mini-2025-08-07-minimal", "cost_per_task": 0.0094 }, { "id": "e57a8ae6c768ee86db29", "model": "o4-mini (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0167, "raw_score": 0.0167, "raw_scale": 1, "source_record": "o4-mini-2025-04-16-low", "cost_per_task": 0.05 }, { "id": "9b538b67b51e63fd8e8d", "model": "GPT-6 Sol (None)", "organization": "OpenAI", "configuration": "CoT", "score": 0.016666666666666666, "raw_score": 0.016666666666666666, "raw_scale": 1, "source_record": "openai-gpt-6-sol-none", "cost_per_task": 0.0740642 }, { "id": "65cf678289fbccbca3ff", "model": "GPT-5.4 Nano (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0153, "raw_score": 0.0153, "raw_scale": 1, "source_record": "gpt-5-4-nano-low", "cost_per_task": 0.01 }, { "id": "641ef7b70caf7453e669", "model": "Gemini 3.5 Flash-Lite (Low)", "organization": "Google", "configuration": "CoT", "score": 0.015277777777777777, "raw_score": 0.015277777777777777, "raw_scale": 1, "source_record": "gemini-3-5-flash-lite-low", "cost_per_task": 0.017051445000000005 }, { "id": "32b282f88c08b524d7aa", "model": "Nemotron 3.5 Super VL (High)", "organization": "NVIDIA", "configuration": "CoT", "score": 0.015277777777777777, "raw_score": 0.015277777777777777, "raw_scale": 1, "source_record": "nvidia-nemotron-3-5-super-vl-120b-a12b-high", "cost_per_task": 0.043333685 }, { "id": "22cda0c1c6d0f8947431", "model": "Qwen3.8-27B (None)", "organization": "Alibaba", "configuration": "CoT", "score": 0.015277777777777777, "raw_score": 0.015277777777777777, "raw_scale": 1, "source_record": "alibaba-qwen3-8-27b-none", "cost_per_task": 0.05891859291666669 }, { "id": "a791366f6f6201254edc", "model": "Deepseek R1", "organization": "DeepSeek", "configuration": "CoT", "score": 0.013, "raw_score": 0.013, "raw_scale": 1, "source_record": "R1", "cost_per_task": 0.08 }, { "id": "a0089ee0749ba5639d0b", "model": "Gemini 2.0 Flash", "organization": "Google", "configuration": "Base LLM", "score": 0.013, "raw_score": 0.013, "raw_scale": 1, "source_record": "Gemini 2.0 Flash", "cost_per_task": 0.004 }, { "id": "2b453e4159634d29f0cd", "model": "Claude Opus 4", "organization": "Anthropic", "configuration": "CoT", "score": 0.0127, "raw_score": 0.0127, "raw_scale": 1, "source_record": "claude-opus-4-20250514", "cost_per_task": 0.6388 }, { "id": "8cb50e5cf14a42a30f0f", "model": "Claude Sonnet 4", "organization": "Anthropic", "configuration": "Base LLM", "score": 0.0127, "raw_score": 0.0127, "raw_scale": 1, "source_record": "claude-sonnet-4-20250514", "cost_per_task": 0.1272 }, { "id": "cda71965296f74cdd864", "model": "Codex Mini (Latest)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0127, "raw_score": 0.0127, "raw_scale": 1, "source_record": "codex-mini-latest", "cost_per_task": 0.23 }, { "id": "09d0ceca06974190cf3c", "model": "Claude Haiku 4.5", "organization": "Anthropic", "configuration": "Base LLM", "score": 0.0125, "raw_score": 0.0125, "raw_scale": 1, "source_record": "claude-haiku-4-5-20251001", "cost_per_task": 0.0426 }, { "id": "ca6419bd74f299132a27", "model": "Claude Haiku 4.5 (Thinking 1K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0125, "raw_score": 0.0125, "raw_scale": 1, "source_record": "claude-haiku-4-5-20251001-thinking-1k", "cost_per_task": 0.0471 }, { "id": "a332123c4d41ea5a8750", "model": "DeepSeek V4.1 Flash (None)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.0125, "raw_score": 0.0125, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-none", "cost_per_task": 0.010176125000000005 }, { "id": "2f1ffc216f00f9b28a51", "model": "Gemini 3 Flash Preview (Low)", "organization": "Google", "configuration": "CoT", "score": 0.0125, "raw_score": 0.0125, "raw_scale": 1, "source_record": "gemini-3-flash-preview-thinking-low", "cost_per_task": 0.0249 }, { "id": "b19a6f6a1206adf2d855", "model": "Qwen3-235b-a22b Instruct (25/07)", "organization": "Alibaba", "configuration": "Base LLM", "score": 0.0125, "raw_score": 0.0125, "raw_scale": 1, "source_record": "qwen3-235b-a22b-instruct-2507", "cost_per_task": 0.0044 }, { "id": "715eca063cb703ff3182", "model": "Deepseek R1 (05/28)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.0112, "raw_score": 0.0112, "raw_scale": 1, "source_record": "deepseek_r1_0528-openrouter", "cost_per_task": 0.0527 }, { "id": "e084898c81741f004d94", "model": "GPT-5.4 Mini (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0111, "raw_score": 0.0111, "raw_scale": 1, "source_record": "gpt-5-4-mini-low", "cost_per_task": 0.06 }, { "id": "5caa805c890382599ebe", "model": "Claude 3.7 (8K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.009, "raw_score": 0.009, "raw_scale": 1, "source_record": "Claude 3.7 Thinking 8K", "cost_per_task": 0.36 }, { "id": "220b2af522752bb012a5", "model": "GPT-5 Nano (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0088, "raw_score": 0.0088, "raw_scale": 1, "source_record": "gpt-5-nano-2025-08-07-medium", "cost_per_task": 0.0137 }, { "id": "2257dc7edab983340f6b", "model": "Claude Sonnet 4 (Thinking 1K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0085, "raw_score": 0.0085, "raw_scale": 1, "source_record": "claude-sonnet-4-20250514-thinking-1k", "cost_per_task": 0.1425 }, { "id": "f03db5ddc60fc3c4e06a", "model": "DeepSeek V4 Pro 0813 (None)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.008333333333333333, "raw_score": 0.008333333333333333, "raw_scale": 1, "source_record": "deepseek-v4-pro-0813-none", "cost_per_task": 0.040291977000000014 }, { "id": "f440cb75932deb83d2b0", "model": "GPT-5.6 Luna 2026-07-30 (None)", "organization": "OpenAI", "configuration": "CoT", "score": 0.008333333333333333, "raw_score": 0.008333333333333333, "raw_scale": 1, "source_record": "openai-gpt-5-6-luna-none-2026-07-30", "cost_per_task": 0.008186350000000002 }, { "id": "2cf3820ec02010bd89b0", "model": "Gemini 3.5 Flash-Lite (Minimal)", "organization": "Google", "configuration": "CoT", "score": 0.008333333333333333, "raw_score": 0.008333333333333333, "raw_scale": 1, "source_record": "gemini-3-5-flash-lite-minimal", "cost_per_task": 0.014339924166666665 }, { "id": "361685e431ef6e4d5a41", "model": "Nemotron 3.5 Super VL (None)", "organization": "NVIDIA", "configuration": "CoT", "score": 0.008333333333333333, "raw_score": 0.008333333333333333, "raw_scale": 1, "source_record": "nvidia-nemotron-3-5-super-vl-120b-a12b-none", "cost_per_task": 0.006350055833333334 }, { "id": "d5de68842a4e9561c8e9", "model": "GPT-5 Mini (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0083, "raw_score": 0.0083, "raw_scale": 1, "source_record": "gpt-5-mini-2025-08-07-low", "cost_per_task": 0.0189 }, { "id": "59d2ee528675972a9f84", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Base LLM", "score": 0.0083, "raw_score": 0.0083, "raw_scale": 1, "source_record": "gpt-5-2-2025-12-11-thinking-none", "cost_per_task": 0.0818 }, { "id": "51c230f9d49a74b10cd1", "model": "o1-mini", "organization": "OpenAI", "configuration": "CoT", "score": 0.0083, "raw_score": 0.0083, "raw_scale": 1, "source_record": "o1-mini-2024-09-12", "cost_per_task": 0.1907 }, { "id": "204dbcd83eba79b03703", "model": "GPT-4.5", "organization": "OpenAI", "configuration": "Base LLM", "score": 0.008, "raw_score": 0.008, "raw_scale": 1, "source_record": "gpt-4-5-2025-02-27", "cost_per_task": 2.1 }, { "id": "24f0dc5fe3260c9fd6d6", "model": "Gemini 1.5 Pro", "organization": "Google", "configuration": "Base LLM", "score": 0.008, "raw_score": 0.008, "raw_scale": 1, "source_record": "Gemini 1.5 Pro", "cost_per_task": 0.04 }, { "id": "a09710171d7774b38a42", "model": "Claude 3.7 (16K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.007, "raw_score": 0.007, "raw_scale": 1, "source_record": "Claude 3.7 Thinking 16K", "cost_per_task": 0.51 }, { "id": "f6cafd6ecbfc01af2118", "model": "GPT-4.1", "organization": "OpenAI", "configuration": "Base LLM", "score": 0.0042, "raw_score": 0.0042, "raw_scale": 1, "source_record": "gpt-4-1-2025-04-14", "cost_per_task": 0.0691 }, { "id": "c07d55df1573e1ed7240", "model": "GPT-5.1 (Thinking, None)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0042, "raw_score": 0.0042, "raw_scale": 1, "source_record": "gpt-5-1-2025-11-13-thinking-none", "cost_per_task": 0.0581 }, { "id": "e4609511bdaa60f75600", "model": "Grok 3 Mini (Low)", "organization": "xAI", "configuration": "CoT", "score": 0.0042, "raw_score": 0.0042, "raw_scale": 1, "source_record": "grok-3-mini-beta-low-openrouter", "cost_per_task": 0.0131 }, { "id": "1e69524fbdd2c6e1ed52", "model": "Claude 3.7 (1K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.004, "raw_score": 0.004, "raw_scale": 1, "source_record": "Claude 3.7 Thinking 1K", "cost_per_task": 0.14 }, { "id": "cd805d6553825e7d93e2", "model": "Claude 3.7", "organization": "Anthropic", "configuration": "Base LLM", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "Claude 3.7", "cost_per_task": 0.12 }, { "id": "d1bf818ebeb4b3afa725", "model": "Claude Opus 4 (Thinking 1K)", "organization": "Anthropic", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "claude-opus-4-20250514-thinking-1k", "cost_per_task": 0.7503 }, { "id": "7d9ccd2b8f2dce4a0055", "model": "GPT-4.1-Mini", "organization": "OpenAI", "configuration": "Base LLM", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "gpt-4-1-mini-2025-04-14", "cost_per_task": 0.0139 }, { "id": "6d77f9ae47bf27c14063", "model": "GPT-4.1-Nano", "organization": "OpenAI", "configuration": "Base LLM", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "gpt-4-1-nano-2025-04-14", "cost_per_task": 0.0036 }, { "id": "247c298f06e40daf5a59", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Base LLM", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "gpt-4o-2024-11-20", "cost_per_task": 0.08 }, { "id": "27bebe765c6e794efc03", "model": "GPT-4o-mini", "organization": "OpenAI", "configuration": "Base LLM", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "gpt-4o-mini-2024-07-18", "cost_per_task": 0.01 }, { "id": "0d1217e789e92c559a0f", "model": "GPT-5 (Minimal)", "organization": "OpenAI", "configuration": "Base LLM", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "gpt-5-2025-08-07-minimal", "cost_per_task": 0.0562 }, { "id": "a15e6ffb779bdd87165d", "model": "GPT-5 Nano (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "gpt-5-nano-2025-08-07-low", "cost_per_task": 0.0033 }, { "id": "022049b78b3a065475c7", "model": "GPT-5 Nano (Minimal)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "gpt-5-nano-2025-08-07-minimal", "cost_per_task": 0.0025 }, { "id": "3b753720bf1e6dfa98d1", "model": "GPT-6 Luna (None)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "openai-gpt-6-luna-none", "cost_per_task": 0.0036990350000000003 }, { "id": "6db5f9ed15592dcc2db9", "model": "Gemini 2.5 Pro (Thinking 1K)", "organization": "Google", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "gemini-2-5-pro-2025-06-17-thinking-1k", "cost_per_task": 0.0885 }, { "id": "fdca675059dd80330ddf", "model": "Grok 3", "organization": "xAI", "configuration": "Base LLM", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "grok-3-openrouter", "cost_per_task": 0.1421 }, { "id": "d7b6b00a54e6887b27bf", "model": "Inkling Small (None)", "organization": "Thinking Machines", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "thinky-inkling-small-none", "cost_per_task": 0.010745875000000005 }, { "id": "807fced31fd3f5d2ef23", "model": "Llama 4 Maverick", "organization": "Meta", "configuration": "Base LLM", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "Llama-4-Maverick-17B-128E-Instruct-FP8-together", "cost_per_task": 0.0121 }, { "id": "42968fe61100f1f36d65", "model": "Llama 4 Scout", "organization": "Meta", "configuration": "Base LLM", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "Llama-4-Scout-17B-16E-Instruct-together", "cost_per_task": 0.0062 }, { "id": "b1a51bc87eb91b0a930a", "model": "Magistral Medium", "organization": "Mistral", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "magistral-medium-2506", "cost_per_task": 0.1079 }, { "id": "fdfcda95eb8ad9d42346", "model": "Magistral Medium (Thinking)", "organization": "Mistral", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "magistral-medium-2506-thinking", "cost_per_task": 0.123 }, { "id": "a9cff9a8d02d69b95961", "model": "Magistral Small", "organization": "Mistral", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "magistral-small-2506", "cost_per_task": 0.0488 }, { "id": "0af9cf4c89daa8c2d91c", "model": "o3-mini (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "o3-mini-2025-01-31-low", "cost_per_task": 0.0623 } ] }, { "id": "frontiermath-tiers-1-3-v2", "name": "FrontierMath Tiers 1–3 (v2)", "group": "数学与科学", "unit": "%", "metric": "mean_score", "protocol": { "en": "Epoch AI evaluations; mean score for the named tier and version. Model and reasoning configuration names follow the source.", "zh": "Epoch AI 测评，使用对应难度层级和版本的平均成绩；模型及推理配置名称沿用原始记录。" }, "source": { "name": "Epoch AI", "url": "https://epoch.ai/frontiermath", "data_url": "https://epoch.ai/data/benchmark_data.zip", "retrieved_at": "2026-11-01T16:45:15.372902+00:00", "sha256": "c4683724f4e6c76fef1b5fedb63726938cfdde1a241eba20fec3e7c3896dcde4", "selector": "frontiermath_tiers_1_3_v2.csv: mean_score", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "status": "current_snapshot", "kind": "evaluator" }, "results": [ { "id": "1504f56dcab54be787d8", "model": "gpt-6-astra_max", "organization": "OpenAI", "configuration": "", "score": 0.9368421052631579, "raw_score": 0.9368421052631579, "raw_scale": 1, "source_record": "csK2fe5T8p2R7WgoJUu6bZ", "evaluated_at": "2026-08-30T14:57:43.000Z" }, { "id": "74949604eb4202dc2608", "model": "gpt-6.1-sol_max", "organization": "OpenAI", "configuration": "", "score": 0.9368421052631579, "raw_score": 0.9368421052631579, "raw_scale": 1, "source_record": "aBoQLaqnXsLPrsbN8wYgUp", "evaluated_at": "2026-09-29T22:00:34.000Z" }, { "id": "160b96d92975877154bc", "model": "claude-opus-5-5_max", "organization": "Anthropic", "configuration": "", "score": 0.9122807017543859, "raw_score": 0.9122807017543859, "raw_scale": 1, "source_record": "LPxhvBNEn7S3xkRZUgcvc9", "evaluated_at": "2026-09-22T18:27:39.000Z" }, { "id": "faf18df03fd892c0082c", "model": "gpt-6.1-terra_max", "organization": "OpenAI", "configuration": "", "score": 0.9087719298245615, "raw_score": 0.9087719298245615, "raw_scale": 1, "source_record": "KkEoE6hBF8RB4ff43Xv63e", "evaluated_at": "2026-10-26T20:00:37.000Z" }, { "id": "9eb2efc656269820e012", "model": "claude-fable-5-1_max", "organization": "Anthropic", "configuration": "", "score": 0.9017543859649123, "raw_score": 0.9017543859649123, "raw_scale": 1, "source_record": "g7dXwZH2tWvqLg3X5idGs2", "evaluated_at": "2026-09-01T18:38:24.000Z" }, { "id": "b6e2a90e6549a63699ef", "model": "gpt-6-sol_max", "organization": "OpenAI", "configuration": "", "score": 0.8982456140350877, "raw_score": 0.8982456140350877, "raw_scale": 1, "source_record": "RHjHEPHEsWZqFuMjBTeC6r", "evaluated_at": "2026-09-22T19:59:24.000Z" }, { "id": "c338316d960dae02e8c2", "model": "gpt-5.6-sol_max", "organization": "OpenAI", "configuration": "", "score": 0.8912280701754386, "raw_score": 0.8912280701754386, "raw_scale": 1, "source_record": "VBpT2WWYKBZtUxG9Z3svGh", "evaluated_at": "2026-07-09T02:45:09.000Z" }, { "id": "84de840d68ef7710bbed", "model": "claude-sonnet-5-5_max", "organization": "Anthropic", "configuration": "", "score": 0.887719298245614, "raw_score": 0.887719298245614, "raw_scale": 1, "source_record": "4tZWwV9tqEgsCaKeSuSoG4", "evaluated_at": "2026-09-29T15:15:14.000Z" }, { "id": "bfb59ecc50330f57c95c", "model": "gpt-5.5-pro_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.8771929824561403, "raw_score": 0.8771929824561403, "raw_scale": 1, "source_record": "LSKDetgzyVojru45w7RJhf", "evaluated_at": "2026-06-12T21:33:12.000Z" }, { "id": "b1ab18cc4eb85d6d1446", "model": "claude-fable-5_max", "organization": "Anthropic", "configuration": "", "score": 0.8701754385964913, "raw_score": 0.8701754385964913, "raw_scale": 1, "source_record": "E28uZiJvrPgKtawbJpoZQ8", "evaluated_at": "2026-06-09T18:12:35.000Z" }, { "id": "fd67a7aad4dbdffeb6f2", "model": "gemini-4-argon_high", "organization": "Google DeepMind", "configuration": "", "score": 0.8596491228070176, "raw_score": 0.8596491228070176, "raw_scale": 1, "source_record": "N8daAyKM6Sah39YBcPRi4W", "evaluated_at": "2026-10-13T17:14:57.000Z" }, { "id": "a9a38fa09fff4fe6cbe7", "model": "gpt-5.6-terra_max", "organization": "OpenAI", "configuration": "", "score": 0.8596491228070176, "raw_score": 0.8596491228070176, "raw_scale": 1, "source_record": "J6evjTMgHkrun8txnEvzdN", "evaluated_at": "2026-07-09T15:58:30.000Z" }, { "id": "c9b730c51115c9cccb03", "model": "claude-opus-5_max", "organization": "Anthropic", "configuration": "", "score": 0.856140350877193, "raw_score": 0.856140350877193, "raw_scale": 1, "source_record": "ff7GeS7rdperjxSBZ7MHZY", "evaluated_at": "2026-07-24T19:54:20.000Z" }, { "id": "db6d2324a6aa10e3bde2", "model": "gpt-5.5_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.8526315789473684, "raw_score": 0.8526315789473684, "raw_scale": 1, "source_record": "5FGXKvXQwGPtcCX2PNrLR5", "evaluated_at": "2026-06-11T17:47:36.000Z" }, { "id": "3b1d1bf9e7f2fd810e8e", "model": "gpt-5.4-pro-2026-03-05_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.8245614035087719, "raw_score": 0.8245614035087719, "raw_scale": 1, "source_record": "ZFVa8dw7oiir6PhcF8XZMo", "evaluated_at": "2026-06-13T02:38:02.000Z" }, { "id": "65fd94892b14a92b9a88", "model": "gpt-5.6-luna_max", "organization": "OpenAI", "configuration": "", "score": 0.8210526315789474, "raw_score": 0.8210526315789474, "raw_scale": 1, "source_record": "YLAk5J3Wd8EruVXee8bAQb", "evaluated_at": "2026-07-09T14:18:35.000Z" }, { "id": "272ce402abbc1126b204", "model": "muse-wonder-1.0_max", "organization": "Meta AI", "configuration": "", "score": 0.8210526315789474, "raw_score": 0.8210526315789474, "raw_scale": 1, "source_record": "hwVgELmPofPY3Vqm7BRLxs", "evaluated_at": "2026-10-12T19:00:20.000Z" }, { "id": "fa1fa7acf90cc2f9fcf2", "model": "claude-opus-4-8_max", "organization": "Anthropic", "configuration": "", "score": 0.8, "raw_score": 0.8, "raw_scale": 1, "source_record": "GVmgtgKMwEqc54f3jA7auP", "evaluated_at": "2026-06-10T15:25:15.000Z" }, { "id": "fcb218fdac50f81b0127", "model": "gpt-6-luna_max", "organization": "OpenAI", "configuration": "", "score": 0.7894736842105263, "raw_score": 0.7894736842105263, "raw_scale": 1, "source_record": "iBQxysYR7zPxB6hxvJyTLc", "evaluated_at": "2026-09-22T22:14:05.000Z" }, { "id": "62fda13cc76a01912423", "model": "gpt-5.4-2026-03-05_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.7859649122807018, "raw_score": 0.7859649122807018, "raw_scale": 1, "source_record": "PaYEQ37A2n6giYBwZGBnXY", "evaluated_at": "2026-06-11T17:55:45.000Z" }, { "id": "a937216741d225ceea4a", "model": "claude-haiku-5-5_max", "organization": "Anthropic", "configuration": "", "score": 0.7508771929824561, "raw_score": 0.7508771929824561, "raw_scale": 1, "source_record": "2QAt4ptgGkLtcbVL3hdAvL", "evaluated_at": "2026-10-07T20:42:31.000Z" }, { "id": "f7f8f4b7133d9d8b61e5", "model": "qwen3.8-max_xhigh", "organization": "Alibaba", "configuration": "", "score": 0.7473684210526316, "raw_score": 0.7473684210526316, "raw_scale": 1, "source_record": "ABT89ieSWGnGj4RYMduHiG", "evaluated_at": "2026-08-04T19:32:51.000Z" }, { "id": "a50ee1e18b4e3179c6a2", "model": "muse-spark-1.3_xhigh", "organization": "Meta AI", "configuration": "", "score": 0.743859649122807, "raw_score": 0.743859649122807, "raw_scale": 1, "source_record": "JeYksd3f2RkegmxCcZyBMQ", "evaluated_at": "2026-09-16T22:00:25.000Z" }, { "id": "bec036391ccb1700875f", "model": "muse-spark-1.3_max", "organization": "Meta AI", "configuration": "", "score": 0.7403508771929824, "raw_score": 0.7403508771929824, "raw_scale": 1, "source_record": "6735DShGR4Zu5KZ2U6n4pC", "evaluated_at": "2026-09-18T16:34:54.000Z" }, { "id": "a047f8f37186dc424eb6", "model": "gpt-5.2-pro-2025-12-11_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.74, "raw_score": 0.74, "raw_scale": 1, "source_record": "dMJenf5xYWu7Pwn2U6w4dW", "evaluated_at": "2026-06-13T02:37:55.000Z" }, { "id": "54b8855f253ac8423b2b", "model": "kimi-k3_max", "organization": "Moonshot", "configuration": "", "score": 0.721830985915493, "raw_score": 0.721830985915493, "raw_scale": 1, "source_record": "bcntamvGsSH6QgETdoaSph", "evaluated_at": "2026-07-17T01:48:42.000Z" }, { "id": "9393ae2c7ad38fec39eb", "model": "gemini-3.7-flash_high", "organization": "Google DeepMind", "configuration": "", "score": 0.7157894736842105, "raw_score": 0.7157894736842105, "raw_scale": 1, "source_record": "axMdgNDC8UYYLnZYLHZpF5", "evaluated_at": "2026-08-14T19:59:58.000Z" }, { "id": "80c47dc7416bd2058820", "model": "deepseek-v4.1-flash-1017_max", "organization": "DeepSeek", "configuration": "", "score": 0.712280701754386, "raw_score": 0.712280701754386, "raw_scale": 1, "source_record": "UnaMiD4Yy8W4WzyeSRwybr", "evaluated_at": "2026-10-19T22:39:00.000Z" }, { "id": "9f0e004fc8a39daddc86", "model": "claude-opus-4-7_max", "organization": "Anthropic", "configuration": "", "score": 0.7017543859649122, "raw_score": 0.7017543859649122, "raw_scale": 1, "source_record": "ZkNQ7GER7Mc9FSPmpUzgKH", "evaluated_at": "2026-06-10T21:41:54.000Z" }, { "id": "ca2557d92fc26a11cfbe", "model": "Muse-Spark-Open-816B-A42B_xhigh", "organization": "Meta AI", "configuration": "", "score": 0.6982456140350877, "raw_score": 0.6982456140350877, "raw_scale": 1, "source_record": "e3e3h9r7Fouu4wsAeVzTnV", "evaluated_at": "2026-10-15T18:52:04.000Z" }, { "id": "f88872b60c3eb75b9ef4", "model": "glm-5.3_max", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.6877192982456141, "raw_score": 0.6877192982456141, "raw_scale": 1, "source_record": "iXvqUzwxAiA3Rg3dwnutFs", "evaluated_at": "2026-08-25T14:20:20.000Z" }, { "id": "a887f115c444e7ca57a6", "model": "gemini-3.8-flash_high", "organization": "Google DeepMind", "configuration": "", "score": 0.6842105263157895, "raw_score": 0.6842105263157895, "raw_scale": 1, "source_record": "Utgs6DuaQDDdi98ZWH22qL", "evaluated_at": "2026-09-02T17:15:36.000Z" }, { "id": "942106285f6b3edcdb98", "model": "gpt-5.2-2025-12-11_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.674, "raw_score": 0.674, "raw_scale": 1, "source_record": "PeZ84GTQdSYQzDxRMWA2Ha", "evaluated_at": "2026-06-11T02:48:30.000Z" }, { "id": "078aee7f64bd5b481ea0", "model": "deepseek-v4.1-flash_max", "organization": "DeepSeek", "configuration": "", "score": 0.6736842105263158, "raw_score": 0.6736842105263158, "raw_scale": 1, "source_record": "b6dXtaGSwvfRYADKYnj9Ci", "evaluated_at": "2026-10-08T19:02:32.000Z" }, { "id": "ff48c21bcd82024015b4", "model": "claude-opus-4-6_max", "organization": "Anthropic", "configuration": "", "score": 0.6596491228070176, "raw_score": 0.6596491228070176, "raw_scale": 1, "source_record": "kgJ2d6XbKj3dZutDcBYSKU", "evaluated_at": "2026-06-11T00:19:44.000Z" }, { "id": "ec87a4618738e9690c3e", "model": "grok-4.6_xhigh", "organization": "xAI", "configuration": "", "score": 0.6596491228070176, "raw_score": 0.6596491228070176, "raw_scale": 1, "source_record": "Zb2AKwa942D7dBTTcebAba", "evaluated_at": "2026-08-14T15:46:50.000Z" }, { "id": "41abc88532c4818b1dee", "model": "claude-sonnet-5_max", "organization": "Anthropic", "configuration": "", "score": 0.656140350877193, "raw_score": 0.656140350877193, "raw_scale": 1, "source_record": "V6YuGpcu8MCbJNXKoJMA6b", "evaluated_at": "2026-06-30T21:54:08.000Z" }, { "id": "797ce9e018f77cdd8691", "model": "qwen3.8-max-0902_xhigh", "organization": "Alibaba", "configuration": "", "score": 0.656140350877193, "raw_score": 0.656140350877193, "raw_scale": 1, "source_record": "FpQf5xBwJPb4ZEo6SgdKWq", "evaluated_at": "2026-09-02T18:04:58.000Z" }, { "id": "cdb94b73f51c90c39cd6", "model": "deepseek-v4-pro-0813_max", "organization": "DeepSeek", "configuration": "", "score": 0.6456140350877193, "raw_score": 0.6456140350877193, "raw_scale": 1, "source_record": "b6yWmeFgsWeEiG9Qzs3NSf", "evaluated_at": "2026-08-19T13:35:48.000Z" }, { "id": "2c7b606a65d398345265", "model": "qwen3.7-max", "organization": "Alibaba", "configuration": "", "score": 0.6456140350877193, "raw_score": 0.6456140350877193, "raw_scale": 1, "source_record": "RG2biwnmGybSVTVxxMdRWa", "evaluated_at": "2026-06-13T03:08:22.000Z" }, { "id": "b9304970a973d756846b", "model": "gemini-3.5-flash_high", "organization": "Google DeepMind", "configuration": "", "score": 0.6280701754385964, "raw_score": 0.6280701754385964, "raw_scale": 1, "source_record": "72DQVwbYiQW5pjhshjBaGU", "evaluated_at": "2026-06-10T17:32:38.000Z" }, { "id": "f3c0522cd0df8dd8dea8", "model": "gemini-3.1-pro-preview", "organization": "Google DeepMind", "configuration": "", "score": 0.5964912280701754, "raw_score": 0.5964912280701754, "raw_scale": 1, "source_record": "hVifTUdQZrFEYabgCRu8Qq", "evaluated_at": "2026-06-11T23:35:57.000Z" }, { "id": "0284a97f5f0670fa1673", "model": "glm-5.2_max", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.592057761732852, "raw_score": 0.592057761732852, "raw_scale": 1, "source_record": "TTuCPacMDpNaCSs3Hpbdjp", "evaluated_at": "2026-06-19T16:13:05.000Z" }, { "id": "192b6ce5fabbc6cdca40", "model": "gemini-3.6-flash_high", "organization": "Google DeepMind", "configuration": "", "score": 0.5894736842105263, "raw_score": 0.5894736842105263, "raw_scale": 1, "source_record": "nb9LnDeeQyvRCqsbsTANGD", "evaluated_at": "2026-08-02T00:36:45.000Z" }, { "id": "9287ce2ae8ce5aa174b0", "model": "mistral-large-4-0_high", "organization": "Mistral AI", "configuration": "", "score": 0.5859649122807018, "raw_score": 0.5859649122807018, "raw_scale": 1, "source_record": "CAstp8hQG6Aehvtw85bMDX", "evaluated_at": "2026-10-12T21:38:35.000Z" }, { "id": "b3219fb2c8c5942e73a6", "model": "deepseek-v4-flash-0731_max", "organization": "DeepSeek", "configuration": "", "score": 0.5754385964912281, "raw_score": 0.5754385964912281, "raw_scale": 1, "source_record": "mKhVMEicZeGjdvJdTRd7mK", "evaluated_at": "2026-08-02T02:15:37.000Z" }, { "id": "e93eeb405b14b239b2d7", "model": "grok-4.5_high", "organization": "xAI", "configuration": "", "score": 0.5719298245614035, "raw_score": 0.5719298245614035, "raw_scale": 1, "source_record": "djbbEqvBW9qux3pZwcZN2i", "evaluated_at": "2026-07-09T01:53:42.000Z" }, { "id": "e60c18cd8d47d81dae96", "model": "kimi-k2.6", "organization": "Moonshot", "configuration": "", "score": 0.5719298245614035, "raw_score": 0.5719298245614035, "raw_scale": 1, "source_record": "BxrNfyMffkVUq4n7U5eK3Q", "evaluated_at": "2026-06-10T19:47:36.000Z" }, { "id": "09837881b9123960179b", "model": "glm-5.3-flash_max", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.5578947368421052, "raw_score": 0.5578947368421052, "raw_scale": 1, "source_record": "YC2u9wykbYQg4gdkf76j26", "evaluated_at": "2026-08-27T13:29:52.000Z" }, { "id": "25dc1c80bf80ba03c8ce", "model": "gpt-5-pro-2025-10-06_high", "organization": "OpenAI", "configuration": "", "score": 0.5578947368421052, "raw_score": 0.5578947368421052, "raw_scale": 1, "source_record": "TFDQUeRKbWiEodRQRrKdpy", "evaluated_at": "2026-06-12T20:32:51.000Z" }, { "id": "2ef4ca3199ecf3766657", "model": "gpt-5-2025-08-07_high", "organization": "OpenAI", "configuration": "", "score": 0.5543859649122806, "raw_score": 0.5543859649122806, "raw_scale": 1, "source_record": "QRDHTGX3rYrVCjVPRarjjt", "evaluated_at": "2026-06-10T20:10:57.000Z" }, { "id": "db9e3481fe0cb3173b15", "model": "glm-5.2_low", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.5473684210526316, "raw_score": 0.5473684210526316, "raw_scale": 1, "source_record": "UnKHyc2idtTmoTUGxGeNJV", "evaluated_at": "2026-08-29T11:07:19.000Z" }, { "id": "4c4cf39c15b2ff0963df", "model": "kimi-k2.7-code", "organization": "Moonshot", "configuration": "", "score": 0.5403508771929825, "raw_score": 0.5403508771929825, "raw_scale": 1, "source_record": "X3RxRkt2pBUXYJjAQHsUo6", "evaluated_at": "2026-06-13T01:39:44.000Z" }, { "id": "30fabac314d7629e99c9", "model": "grok-4.7_xhigh", "organization": "xAI", "configuration": "", "score": 0.5298245614035088, "raw_score": 0.5298245614035088, "raw_scale": 1, "source_record": "F4unBbW5M4VJogzCq4vGAb", "evaluated_at": "2026-09-22T21:49:22.000Z" }, { "id": "0278d049d982cdd80f2c", "model": "gemini-3-flash-preview", "organization": "Google DeepMind", "configuration": "", "score": 0.512280701754386, "raw_score": 0.512280701754386, "raw_scale": 1, "source_record": "EtgrDq6ei6kZe8K7QbXUe3", "evaluated_at": "2026-06-11T23:44:30.000Z" }, { "id": "8513be5f5262da30e933", "model": "gpt-5.4-mini-2026-03-17_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.512280701754386, "raw_score": 0.512280701754386, "raw_scale": 1, "source_record": "jjFRrQvsud3XsgYufr9f8f", "evaluated_at": "2026-06-12T16:40:38.000Z" }, { "id": "b8dd7cc799ac4f74cdc0", "model": "gpt-5-mini-2025-08-07_high", "organization": "OpenAI", "configuration": "", "score": 0.4666666666666667, "raw_score": 0.4666666666666667, "raw_scale": 1, "source_record": "YSR9WCR2SnJYvP7DJwX7YN", "evaluated_at": "2026-06-12T16:48:15.000Z" }, { "id": "843338f382259c05f507", "model": "inkling-small_xhigh", "organization": "Thinking Machines", "configuration": "", "score": 0.4631578947368421, "raw_score": 0.4631578947368421, "raw_scale": 1, "source_record": "B6bZbqickq8o2fQcgDmzMo", "evaluated_at": "2026-08-14T19:59:09.000Z" }, { "id": "9f017319db772f6d346d", "model": "deepseek-v4-pro_max", "organization": "DeepSeek", "configuration": "", "score": 0.45263157894736844, "raw_score": 0.45263157894736844, "raw_scale": 1, "source_record": "KUar3FaFjzq5SMKrEhtn8e", "evaluated_at": "2026-06-17T02:30:14.000Z" }, { "id": "66c846ecc7ce0bab2f57", "model": "gpt-5.4-nano-2026-03-17_high", "organization": "OpenAI", "configuration": "", "score": 0.44912280701754387, "raw_score": 0.44912280701754387, "raw_scale": 1, "source_record": "iKtopjTjyCWZqJaEjcc7ZL", "evaluated_at": "2026-06-12T16:48:06.000Z" }, { "id": "3393976dff031f114ca4", "model": "grok-4.20-0309-reasoning", "organization": "xAI", "configuration": "", "score": 0.44912280701754387, "raw_score": 0.44912280701754387, "raw_scale": 1, "source_record": "FLDt53waLTeshC7PK9uVy2", "evaluated_at": "2026-07-13T15:51:35.000Z" }, { "id": "3d076a5929cbf9314506", "model": "grok-4.3_high", "organization": "xAI", "configuration": "", "score": 0.4280701754385965, "raw_score": 0.4280701754385965, "raw_scale": 1, "source_record": "bdsZjrG4ayzP8NAJg5q586", "evaluated_at": "2026-06-17T17:10:48.000Z" }, { "id": "3a86562fa703c1ce4250", "model": "glm-5.2_none", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.4245614035087719, "raw_score": 0.4245614035087719, "raw_scale": 1, "source_record": "ECdG3m8n5ABfWVZ55rPFmT", "evaluated_at": "2026-08-29T11:07:59.000Z" }, { "id": "5f3ee3b42786d4e5a9e9", "model": "gpt-5.6-luna_low", "organization": "OpenAI", "configuration": "", "score": 0.41403508771929826, "raw_score": 0.41403508771929826, "raw_scale": 1, "source_record": "ZryXhNUpPedPvGhbqdNBEw", "evaluated_at": "2026-08-29T11:07:20.000Z" }, { "id": "6d3a13418995e805cd3b", "model": "gpt-5.6-luna_none", "organization": "OpenAI", "configuration": "", "score": 0.39649122807017545, "raw_score": 0.39649122807017545, "raw_scale": 1, "source_record": "YwHKNFQjbat97wVLqachmi", "evaluated_at": "2026-08-29T11:07:39.000Z" }, { "id": "1bdde5b4d658b6aa3927", "model": "qwen3.6-plus", "organization": "Alibaba", "configuration": "", "score": 0.3824561403508772, "raw_score": 0.3824561403508772, "raw_scale": 1, "source_record": "jbyVWnEvvFVNNc8AEF6WcQ", "evaluated_at": "2026-08-30T22:34:20.000Z" }, { "id": "fbd62e044c33f1f89ae4", "model": "gpt-5-2025-08-07_low", "organization": "OpenAI", "configuration": "", "score": 0.3719298245614035, "raw_score": 0.3719298245614035, "raw_scale": 1, "source_record": "J2DWDpGLBhihZx82M3hhug", "evaluated_at": "2026-08-27T19:59:52.000Z" }, { "id": "a0836931b3bc2ee66b0d", "model": "glm-5.1", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.3684210526315789, "raw_score": 0.3684210526315789, "raw_scale": 1, "source_record": "Mhwme9odWJMzMjoYkW8r6B", "evaluated_at": "2026-08-29T11:06:22.000Z" }, { "id": "b8645de4ce341d711b03", "model": "o4-mini-2025-04-16_high", "organization": "OpenAI", "configuration": "", "score": 0.36140350877192984, "raw_score": 0.36140350877192984, "raw_scale": 1, "source_record": "dcmRi5JTa27EuwiQTdqr2B", "evaluated_at": "2026-06-11T13:04:19.278Z" }, { "id": "b9070dac33ebea62e949", "model": "qwen3.6-27b", "organization": "Alibaba", "configuration": "", "score": 0.3508771929824561, "raw_score": 0.3508771929824561, "raw_scale": 1, "source_record": "VonoEhAsGSJ2FHU33qnhui", "evaluated_at": "2026-08-28T12:30:22.000Z" }, { "id": "4cbebcfcc3af91fdde29", "model": "claude-opus-4-5-20251101_32K", "organization": "Anthropic", "configuration": "", "score": 0.34385964912280703, "raw_score": 0.34385964912280703, "raw_scale": 1, "source_record": "mS5EwCRZVpvUvJEehPUDZu", "evaluated_at": "2026-06-11T01:55:25.000Z" }, { "id": "b4528770645efe614529", "model": "qwen3.7-plus_none", "organization": "Alibaba", "configuration": "", "score": 0.34385964912280703, "raw_score": 0.34385964912280703, "raw_scale": 1, "source_record": "Tt8tBXHoCEYqGLePWXkM8u", "evaluated_at": "2026-08-29T11:07:26.000Z" }, { "id": "fc9259a9995ccdd2ea57", "model": "qwen3.6-27b_none", "organization": "Alibaba", "configuration": "", "score": 0.34035087719298246, "raw_score": 0.34035087719298246, "raw_scale": 1, "source_record": "WjXdTPMnJUAT27azUKo3Ka", "evaluated_at": "2026-08-29T11:06:28.000Z" }, { "id": "df74adc948ea54fd71b4", "model": "Inkling_xhigh", "organization": "Thinking Machines", "configuration": "", "score": 0.3333333333333333, "raw_score": 0.3333333333333333, "raw_scale": 1, "source_record": "6KvQvuMMX5zAuteUc6pFcW", "evaluated_at": "2026-08-06T20:14:32.000Z" }, { "id": "7368a36a8a14cb0de228", "model": "o3-2025-04-16_high", "organization": "OpenAI", "configuration": "", "score": 0.3333333333333333, "raw_score": 0.3333333333333333, "raw_scale": 1, "source_record": "N4HerYebFnJDEU4PEvNNGM", "evaluated_at": "2026-08-27T20:00:08.000Z" }, { "id": "6ce97b2dba4f1effdf40", "model": "qwen3.6-plus_none", "organization": "Alibaba", "configuration": "", "score": 0.32280701754385965, "raw_score": 0.32280701754385965, "raw_scale": 1, "source_record": "Z4xfwTke5pR6RrY952bttA", "evaluated_at": "2026-08-29T11:06:40.000Z" }, { "id": "94e4d0f3762434a5ac8b", "model": "qwen3.5-397b-a17b_none", "organization": "Alibaba", "configuration": "", "score": 0.312280701754386, "raw_score": 0.312280701754386, "raw_scale": 1, "source_record": "A4yUA9PfzRr92JAqDDo9ro", "evaluated_at": "2026-08-30T22:34:18.000Z" }, { "id": "489f6f203b767561ced8", "model": "o3-2025-04-16_medium", "organization": "OpenAI", "configuration": "", "score": 0.2982456140350877, "raw_score": 0.2982456140350877, "raw_scale": 1, "source_record": "QV5JgKBrZtfVmNj9fRWJKA", "evaluated_at": "2026-08-27T19:59:58.000Z" }, { "id": "fb73a3a110ec5fd4aa34", "model": "qwen3.5-397b-a17b", "organization": "Alibaba", "configuration": "", "score": 0.29473684210526313, "raw_score": 0.29473684210526313, "raw_scale": 1, "source_record": "EzMaDdXs4YQTMLgppBGxAb", "evaluated_at": "2026-08-28T12:28:11.000Z" }, { "id": "7db819315b44becd968c", "model": "o4-mini-2025-04-16_medium", "organization": "OpenAI", "configuration": "", "score": 0.28771929824561404, "raw_score": 0.28771929824561404, "raw_scale": 1, "source_record": "WorgtXwbnWQSAzMfPgbBH9", "evaluated_at": "2026-08-27T17:53:57.000Z" }, { "id": "73632b7c8cd5e5b56268", "model": "gemini-3.1-flash-lite_high", "organization": "Google", "configuration": "", "score": 0.2771929824561403, "raw_score": 0.2771929824561403, "raw_scale": 1, "source_record": "MVZf59jt9uEoWG9w5W9apB", "evaluated_at": "2026-08-30T22:34:53.000Z" }, { "id": "ad7635991112d4a3e82d", "model": "gpt-5.5-instant", "organization": "OpenAI", "configuration": "", "score": 0.2631578947368421, "raw_score": 0.2631578947368421, "raw_scale": 1, "source_record": "gGcHjE6ppy54ahN2pfxsGH", "evaluated_at": "2026-08-02T03:23:36.000Z" }, { "id": "3ed53b9569eb649f6733", "model": "gemini-3.5-flash-lite_high", "organization": "Google DeepMind", "configuration": "", "score": 0.2596491228070175, "raw_score": 0.2596491228070175, "raw_scale": 1, "source_record": "8o2J9uQLKDCTiCFwYdkJRK", "evaluated_at": "2026-08-02T01:20:17.000Z" }, { "id": "75eedd54998a030cd67f", "model": "glm-5.1_none", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.24912280701754386, "raw_score": 0.24912280701754386, "raw_scale": 1, "source_record": "4hmmfwG9tgZJvKhL78nKzL", "evaluated_at": "2026-08-28T12:30:03.000Z" }, { "id": "cf55ed985777c9a58d8b", "model": "gemini-2.5-pro", "organization": "Google DeepMind", "configuration": "", "score": 0.24561403508771928, "raw_score": 0.24561403508771928, "raw_scale": 1, "source_record": "FkywQ2KS4AkLPzjSPmhHcS", "evaluated_at": "2026-06-11T23:56:12.000Z" }, { "id": "96418c34ba5b3215f3e7", "model": "gpt-5.4-mini-2026-03-17_low", "organization": "OpenAI", "configuration": "", "score": 0.24561403508771928, "raw_score": 0.24561403508771928, "raw_scale": 1, "source_record": "8nsNpE6ETikLf4tSrNHmGs", "evaluated_at": "2026-08-28T12:28:53.000Z" }, { "id": "8ef5f6a80e8204ef69e6", "model": "claude-sonnet-4-5-20250929_32K", "organization": "Anthropic", "configuration": "", "score": 0.23859649122807017, "raw_score": 0.23859649122807017, "raw_scale": 1, "source_record": "7oUkeLfwMK3MtkSNHX6QPu", "evaluated_at": "2026-06-11T21:10:25.000Z" }, { "id": "beaa9e0fb1f33dc22434", "model": "gemini-3.1-flash-lite_low", "organization": "Google", "configuration": "", "score": 0.22456140350877193, "raw_score": 0.22456140350877193, "raw_scale": 1, "source_record": "VcuyQvF8LWuYhQCx2FGDHh", "evaluated_at": "2026-08-30T22:35:03.000Z" }, { "id": "d9d49b570e48f555976a", "model": "qwen3.6-flash_none", "organization": "Alibaba", "configuration": "", "score": 0.22456140350877193, "raw_score": 0.22456140350877193, "raw_scale": 1, "source_record": "RoscWx4NUGVqcHdko6QhKd", "evaluated_at": "2026-08-29T11:06:46.000Z" }, { "id": "d9a248ba0b2e1dd8e1b0", "model": "gemini-3.1-flash-lite_minimal", "organization": "Google", "configuration": "", "score": 0.21403508771929824, "raw_score": 0.21403508771929824, "raw_scale": 1, "source_record": "4gZy2dAAT2EPZDGz5puD3H", "evaluated_at": "2026-08-28T12:55:30.000Z" }, { "id": "dd768853c58693b67ab5", "model": "gpt-5.4-nano-2026-03-17_low", "organization": "OpenAI", "configuration": "", "score": 0.20350877192982456, "raw_score": 0.20350877192982456, "raw_scale": 1, "source_record": "BNSjwbcpNSaDXHF7gC9WoN", "evaluated_at": "2026-08-28T12:28:58.000Z" }, { "id": "b294e7e65b7d4e9f9278", "model": "qwen3.6-35b-a3b_none", "organization": "Alibaba", "configuration": "", "score": 0.20350877192982456, "raw_score": 0.20350877192982456, "raw_scale": 1, "source_record": "YRvn36dmMyDbS5e4XkF4Xi", "evaluated_at": "2026-08-30T22:34:37.000Z" }, { "id": "9a219a92298f462bf97b", "model": "gpt-5-nano-2025-08-07_high", "organization": "OpenAI", "configuration": "", "score": 0.2, "raw_score": 0.2, "raw_scale": 1, "source_record": "JH3PjDrbrQndEcc8fh5Qre", "evaluated_at": "2026-06-12T16:47:59.000Z" }, { "id": "972d90b229e73c5575f2", "model": "o3-2025-04-16_low", "organization": "OpenAI", "configuration": "", "score": 0.19298245614035087, "raw_score": 0.19298245614035087, "raw_scale": 1, "source_record": "CX3BTd8Yir7nQaRaNGWhA4", "evaluated_at": "2026-08-27T17:55:20.000Z" }, { "id": "f63d5e67b5339a600db4", "model": "qwen3.7-flash", "organization": "Alibaba", "configuration": "", "score": 0.19298245614035087, "raw_score": 0.19298245614035087, "raw_scale": 1, "source_record": "6pTYmQCw3EMbErvAqpivH5", "evaluated_at": "2026-08-29T11:07:49.000Z" }, { "id": "43a71d0562f29d61085b", "model": "qwen3.7-flash_none", "organization": "Alibaba", "configuration": "", "score": 0.19298245614035087, "raw_score": 0.19298245614035087, "raw_scale": 1, "source_record": "doNrgGQY5SytQV22WRqw5L", "evaluated_at": "2026-08-29T11:07:51.000Z" }, { "id": "87c23af7c8c908ad9462", "model": "qwen3-max-2025-09-23", "organization": "Alibaba", "configuration": "", "score": 0.18947368421052632, "raw_score": 0.18947368421052632, "raw_scale": 1, "source_record": "NrTH3EYHjp4hPdxgpCPGTX", "evaluated_at": "2026-08-30T22:34:08.000Z" }, { "id": "326585992030bcc0ab40", "model": "o3-mini-2025-01-31_high", "organization": "OpenAI", "configuration": "", "score": 0.18596491228070175, "raw_score": 0.18596491228070175, "raw_scale": 1, "source_record": "mRJaJyBY2M2GGdbdMmT6iq", "evaluated_at": "2026-06-11T13:04:17.668Z" }, { "id": "e5cfc00893c876c3ada6", "model": "gpt-5-2025-08-07_minimal", "organization": "OpenAI", "configuration": "", "score": 0.1824561403508772, "raw_score": 0.1824561403508772, "raw_scale": 1, "source_record": "53PtW8g64r6zeLY38aK9Q4", "evaluated_at": "2026-08-27T17:53:33.000Z" }, { "id": "fc7b72ff2f90f96e1b63", "model": "gpt-5-mini-2025-08-07_low", "organization": "OpenAI", "configuration": "", "score": 0.1824561403508772, "raw_score": 0.1824561403508772, "raw_scale": 1, "source_record": "SrVpkvKRweaBHSVwPLAhBQ", "evaluated_at": "2026-08-27T17:53:29.000Z" }, { "id": "fe618f91cabc216819cb", "model": "qwen3.5-flash_none", "organization": "Alibaba", "configuration": "", "score": 0.1824561403508772, "raw_score": 0.1824561403508772, "raw_scale": 1, "source_record": "afxoicJc4C6caVDwUpsiME", "evaluated_at": "2026-08-28T12:28:39.000Z" }, { "id": "a18810c554514cc18d5c", "model": "qwen3.6-35b-a3b", "organization": "Alibaba", "configuration": "", "score": 0.17543859649122806, "raw_score": 0.17543859649122806, "raw_scale": 1, "source_record": "ggG3n6WVPZn7Gu2RNyfQbK", "evaluated_at": "2026-08-29T11:06:39.000Z" }, { "id": "6e4210861539ad8705e7", "model": "gpt-5.4-mini-2026-03-17_none", "organization": "OpenAI", "configuration": "", "score": 0.17192982456140352, "raw_score": 0.17192982456140352, "raw_scale": 1, "source_record": "gDwrSBE5GvFXymdBYMnMyV", "evaluated_at": "2026-08-28T12:29:04.000Z" }, { "id": "cd6fd936edcd6a3c50d7", "model": "qwen3.6-flash", "organization": "Alibaba", "configuration": "", "score": 0.17192982456140352, "raw_score": 0.17192982456140352, "raw_scale": 1, "source_record": "nDsNFM5ioGzyWcpeUKVotg", "evaluated_at": "2026-08-29T11:07:18.000Z" }, { "id": "460cdaef019862102f27", "model": "o4-mini-2025-04-16_low", "organization": "OpenAI", "configuration": "", "score": 0.16140350877192983, "raw_score": 0.16140350877192983, "raw_scale": 1, "source_record": "CpEZwR38bY8xb29bVmNHFm", "evaluated_at": "2026-08-27T17:54:12.000Z" }, { "id": "4c76c2bd4e77ae295472", "model": "o1-2024-12-17_high", "organization": "OpenAI", "configuration": "", "score": 0.14736842105263157, "raw_score": 0.14736842105263157, "raw_scale": 1, "source_record": "NMJhDtxtyQJCHQ4EApr9LM", "evaluated_at": "2026-08-28T07:09:23.000Z" }, { "id": "baab6c10556385e7f3a1", "model": "claude-opus-4-1-20250805_32K", "organization": "Anthropic", "configuration": "", "score": 0.12631578947368421, "raw_score": 0.12631578947368421, "raw_scale": 1, "source_record": "5eRkmXCURH2pRbhJoukfnC", "evaluated_at": "2026-06-11T21:15:07.000Z" }, { "id": "dbe1a065ab6756a0c733", "model": "o3-mini-2025-01-31_medium", "organization": "OpenAI", "configuration": "", "score": 0.10526315789473684, "raw_score": 0.10526315789473684, "raw_scale": 1, "source_record": "hRYELqPG6rnY5cBtJhYnt7", "evaluated_at": "2026-08-27T17:53:55.000Z" }, { "id": "7e48774b6a1d946b69c6", "model": "o1-2024-12-17_medium", "organization": "OpenAI", "configuration": "", "score": 0.10175438596491228, "raw_score": 0.10175438596491228, "raw_scale": 1, "source_record": "4RnNJhaFMGdLSFjoHHMR8k", "evaluated_at": "2026-08-27T21:00:05.000Z" }, { "id": "ae18e9b43e4dcd671a30", "model": "qwen3.5-flash", "organization": "Alibaba", "configuration": "", "score": 0.09473684210526316, "raw_score": 0.09473684210526316, "raw_scale": 1, "source_record": "6XZAenohp9pCgQ3gVeUAi8", "evaluated_at": "2026-08-28T12:28:30.000Z" }, { "id": "5ebe72e43ace9e5ce66d", "model": "o1-2024-12-17_low", "organization": "OpenAI", "configuration": "", "score": 0.08421052631578947, "raw_score": 0.08421052631578947, "raw_scale": 1, "source_record": "QnMPQZzDRHcTKZRaWYunaa", "evaluated_at": "2026-08-27T21:00:29.000Z" }, { "id": "0cfc80dedc7e803ca865", "model": "gpt-4.1-mini-2025-04-14", "organization": "OpenAI", "configuration": "", "score": 0.06666666666666667, "raw_score": 0.06666666666666667, "raw_scale": 1, "source_record": "6EQrp2MY4QMYLfTxpioYTe", "evaluated_at": "2026-08-27T19:59:44.000Z" }, { "id": "28e1caff36e95dbde902", "model": "gpt-4.1-2025-04-14", "organization": "OpenAI", "configuration": "", "score": 0.05964912280701754, "raw_score": 0.05964912280701754, "raw_scale": 1, "source_record": "kCG2nzspryMy7LBwWPqwXy", "evaluated_at": "2026-08-27T17:54:56.000Z" }, { "id": "5417b6ebd831c75850c2", "model": "gpt-5-mini-2025-08-07_minimal", "organization": "OpenAI", "configuration": "", "score": 0.05964912280701754, "raw_score": 0.05964912280701754, "raw_scale": 1, "source_record": "jrbb2oJv95wiFbnCfN56y9", "evaluated_at": "2026-08-27T17:53:38.000Z" }, { "id": "46e75345714037ad8d13", "model": "gpt-5-nano-2025-08-07_low", "organization": "OpenAI", "configuration": "", "score": 0.05964912280701754, "raw_score": 0.05964912280701754, "raw_scale": 1, "source_record": "kfDcrjxJgjsW8CvG3L8yxz", "evaluated_at": "2026-08-27T17:52:46.000Z" }, { "id": "2d60b9bd53f31f75cfcf", "model": "gpt-5.4-nano-2026-03-17_none", "organization": "OpenAI", "configuration": "", "score": 0.0456140350877193, "raw_score": 0.0456140350877193, "raw_scale": 1, "source_record": "KDQZWiPXB2VXc4bj4GJQW4", "evaluated_at": "2026-08-28T12:29:12.000Z" }, { "id": "b4a73ed1bddcf3ab4bd3", "model": "o3-mini-2025-01-31_low", "organization": "OpenAI", "configuration": "", "score": 0.03859649122807018, "raw_score": 0.03859649122807018, "raw_scale": 1, "source_record": "abYpn5zgPpRdBoYdzuinPt", "evaluated_at": "2026-08-27T17:53:02.000Z" }, { "id": "03f79259042ada84f657", "model": "gpt-5-nano-2025-08-07_minimal", "organization": "OpenAI", "configuration": "", "score": 0.017543859649122806, "raw_score": 0.017543859649122806, "raw_scale": 1, "source_record": "nFt6PoshWAVXE6iH9PQgo2", "evaluated_at": "2026-08-27T17:53:21.000Z" }, { "id": "d73a24abd52d81c859f5", "model": "gpt-4-turbo-2024-04-09", "organization": "OpenAI", "configuration": "", "score": 0.007017543859649123, "raw_score": 0.007017543859649123, "raw_scale": 1, "source_record": "bC2dbN4x2GWWmnnfseKxgQ", "evaluated_at": "2026-08-27T17:54:49.000Z" }, { "id": "ce72f65a9b04a18a33cc", "model": "gpt-4o-mini-2024-07-18", "organization": "OpenAI", "configuration": "", "score": 0.007017543859649123, "raw_score": 0.007017543859649123, "raw_scale": 1, "source_record": "6XhtKr59amKko65T7RUB2a", "evaluated_at": "2026-08-27T19:51:52.000Z" }, { "id": "771262073581f383beb1", "model": "gpt-4o-2024-08-06", "organization": "OpenAI", "configuration": "", "score": 0.0035087719298245615, "raw_score": 0.0035087719298245615, "raw_scale": 1, "source_record": "ikS95CXY2foitnUPDKwNJn", "evaluated_at": "2026-08-27T17:54:29.000Z" }, { "id": "6bb4f158aa6864b8349a", "model": "gpt-3.5-turbo-0125", "organization": "OpenAI", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "FNaBAvmeTiRRFDW63KemZ7", "evaluated_at": "2026-08-27T19:59:25.000Z" } ] }, { "id": "frontiermath-tier-4-v2", "name": "FrontierMath Tier 4 (v2)", "group": "数学与科学", "unit": "%", "metric": "mean_score", "protocol": { "en": "Epoch AI evaluations; mean score for the named tier and version. Model and reasoning configuration names follow the source.", "zh": "Epoch AI 测评，使用对应难度层级和版本的平均成绩；模型及推理配置名称沿用原始记录。" }, "source": { "name": "Epoch AI", "url": "https://epoch.ai/frontiermath", "data_url": "https://epoch.ai/data/benchmark_data.zip", "retrieved_at": "2026-11-01T16:45:15.372902+00:00", "sha256": "c4683724f4e6c76fef1b5fedb63726938cfdde1a241eba20fec3e7c3896dcde4", "selector": "frontiermath_tier_4_v2.csv: mean_score", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "status": "current_snapshot", "kind": "evaluator" }, "results": [ { "id": "3281c0a68d674aacb215", "model": "gpt-6.1-sol_max", "organization": "OpenAI", "configuration": "", "score": 1.0, "raw_score": 1.0, "raw_scale": 1, "source_record": "HgAUm5zxrWbkfYUdeJMGzL", "evaluated_at": "2026-09-29T22:00:34.000Z" }, { "id": "a2e3083db5c566b23ffc", "model": "gpt-6-astra_high", "organization": "OpenAI", "configuration": "", "score": 0.976, "raw_score": 0.976, "raw_scale": 1, "source_record": "f46NQY9jSxyKqo9qzrmk6F", "evaluated_at": "2026-08-30T02:22:12.000Z" }, { "id": "ddf0bf687fadf0a5dbf6", "model": "gpt-6-astra_max", "organization": "OpenAI", "configuration": "", "score": 0.976, "raw_score": 0.976, "raw_scale": 1, "source_record": "f7BCT27LzcAtaLDP2tcrWF", "evaluated_at": "2026-08-30T02:22:12.000Z" }, { "id": "55385695ea23b2e35c69", "model": "gpt-6-astra_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.976, "raw_score": 0.976, "raw_scale": 1, "source_record": "AfKJ2DGMhkjZmeFZxGoY3w", "evaluated_at": "2026-08-30T02:22:12.000Z" }, { "id": "567d1e15c537bca60c05", "model": "gpt-6-astra_medium", "organization": "OpenAI", "configuration": "", "score": 0.975609756097561, "raw_score": 0.975609756097561, "raw_scale": 1, "source_record": "5sNRWemuVwb4adBaSwrLFf", "evaluated_at": "2026-08-30T02:22:12.000Z" }, { "id": "67c9af967cdefc41badf", "model": "claude-opus-5-5_max", "organization": "Anthropic", "configuration": "", "score": 0.95, "raw_score": 0.95, "raw_scale": 1, "source_record": "9qKrTxPhpfW985gieKU56x", "evaluated_at": "2026-09-22T18:27:39.000Z" }, { "id": "d85ec44514d7b7d13320", "model": "gpt-6.1-terra_max", "organization": "OpenAI", "configuration": "", "score": 0.926829268292683, "raw_score": 0.926829268292683, "raw_scale": 1, "source_record": "W2qXKr9YjjCWcEdgvUw93P", "evaluated_at": "2026-10-26T20:00:37.000Z" }, { "id": "1db8e18bcd87a36ed098", "model": "claude-fable-5_max", "organization": "Anthropic", "configuration": "", "score": 0.902, "raw_score": 0.902, "raw_scale": 1, "source_record": "ReDAtJGGfRtrEGGWf4Zbsd", "evaluated_at": "2026-06-09T18:12:35.000Z" }, { "id": "5c92fa7d565ec707f4b4", "model": "gpt-6-sol_max", "organization": "OpenAI", "configuration": "", "score": 0.9, "raw_score": 0.9, "raw_scale": 1, "source_record": "jwS3LYLCPbZpHDzzkX6LyV", "evaluated_at": "2026-09-22T19:59:24.000Z" }, { "id": "85d8c89f74581e324e43", "model": "claude-fable-5-1_max", "organization": "Anthropic", "configuration": "", "score": 0.8780487804878049, "raw_score": 0.8780487804878049, "raw_scale": 1, "source_record": "T56N3X4KLA3xz7tf5aCMGb", "evaluated_at": "2026-09-01T18:38:24.000Z" }, { "id": "ea77939c96f6ea79701d", "model": "gpt-6-astra_low", "organization": "OpenAI", "configuration": "", "score": 0.8780487804878049, "raw_score": 0.8780487804878049, "raw_scale": 1, "source_record": "kRPSnHnDdheajNdqmKsMjs", "evaluated_at": "2026-08-30T02:22:12.000Z" }, { "id": "5f3858c785c380354702", "model": "gpt-5.6-sol_max", "organization": "OpenAI", "configuration": "", "score": 0.8292682926829268, "raw_score": 0.8292682926829268, "raw_scale": 1, "source_record": "QJ5rJMVypB4PMxPcBGftc8", "evaluated_at": "2026-07-09T02:45:09.000Z" }, { "id": "3a5502f338665f34f663", "model": "gpt-6-astra_none", "organization": "OpenAI", "configuration": "", "score": 0.8292682926829268, "raw_score": 0.8292682926829268, "raw_scale": 1, "source_record": "4BCZTu7HQKmhdwRR2W4bzT", "evaluated_at": "2026-08-30T02:22:12.000Z" }, { "id": "2dc0205c354968ff8e88", "model": "claude-sonnet-5-5_max", "organization": "Anthropic", "configuration": "", "score": 0.8048780487804879, "raw_score": 0.8048780487804879, "raw_scale": 1, "source_record": "Xo7ecFEMqPJ4HicvJTvWZZ", "evaluated_at": "2026-09-29T15:15:14.000Z" }, { "id": "1704e36c9495eb6c4cfe", "model": "gpt-5.6-sol_promax", "organization": "OpenAI", "configuration": "", "score": 0.8048780487804879, "raw_score": 0.8048780487804879, "raw_scale": 1, "source_record": "cdWwJSH3jVfrKyhUHmT54T", "evaluated_at": "2026-07-09T20:34:41.000Z" }, { "id": "ba0ef82979fcc6649f94", "model": "gemini-4-argon_high", "organization": "Google DeepMind", "configuration": "", "score": 0.7804878048780488, "raw_score": 0.7804878048780488, "raw_scale": 1, "source_record": "6FQCa8tMipf2ncdgGgMaPc", "evaluated_at": "2026-10-13T17:14:57.000Z" }, { "id": "05b839c33a3788949dea", "model": "gpt-5.5-pro_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.7804878048780488, "raw_score": 0.7804878048780488, "raw_scale": 1, "source_record": "SFKXCPUu73mwxMoUotiZM2", "evaluated_at": "2026-06-12T21:33:12.000Z" }, { "id": "97c853db170796ebe957", "model": "gdm-ai-co-mathematician", "organization": "Google DeepMind", "configuration": "", "score": 0.756, "raw_score": 0.756, "raw_scale": 1, "source_record": "gdm-ai-co-mathematician-manual-run-v2", "evaluated_at": "2026-06-12T14:01:58.023Z" }, { "id": "8cf89f1907fe900b6796", "model": "claude-opus-5_max", "organization": "Anthropic", "configuration": "", "score": 0.7317073170731707, "raw_score": 0.7317073170731707, "raw_scale": 1, "source_record": "ji3kMn49Jcr9t4whxcpjj4", "evaluated_at": "2026-07-24T19:54:20.000Z" }, { "id": "d71bce4cc6cd6d81da62", "model": "gpt-5.5_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.725, "raw_score": 0.725, "raw_scale": 1, "source_record": "PKSMspJiJdNThB6qNvaBSM", "evaluated_at": "2026-06-11T17:47:36.000Z" }, { "id": "95774ebd710f2455deed", "model": "gpt-5.6-terra_max", "organization": "OpenAI", "configuration": "", "score": 0.7073170731707317, "raw_score": 0.7073170731707317, "raw_scale": 1, "source_record": "ZUygpkXqg3NjchXXfedAWV", "evaluated_at": "2026-07-09T15:58:30.000Z" }, { "id": "d06b9c565c6455382ea4", "model": "muse-wonder-1.0_max", "organization": "Meta AI", "configuration": "", "score": 0.7073170731707317, "raw_score": 0.7073170731707317, "raw_scale": 1, "source_record": "76KF9F9owpePuyHpP68mdZ", "evaluated_at": "2026-10-12T19:00:20.000Z" }, { "id": "56a985aa930898f32746", "model": "gpt-5.6-luna_max", "organization": "OpenAI", "configuration": "", "score": 0.6097560975609756, "raw_score": 0.6097560975609756, "raw_scale": 1, "source_record": "NLiiJJSyA9wmg2FHENFpAq", "evaluated_at": "2026-07-09T14:18:35.000Z" }, { "id": "5aaa64645b7f1aceb102", "model": "gpt-5.4-pro-2026-03-05_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.5853658536585366, "raw_score": 0.5853658536585366, "raw_scale": 1, "source_record": "W6b3C8AeNC7CAFs5DQma8N", "evaluated_at": "2026-06-13T02:38:02.000Z" }, { "id": "e705f9a162f34f59992d", "model": "claude-opus-4-8_max", "organization": "Anthropic", "configuration": "", "score": 0.5609756097560976, "raw_score": 0.5609756097560976, "raw_scale": 1, "source_record": "JiqgqBTfkGx797uowZgJbk", "evaluated_at": "2026-06-10T15:25:15.000Z" }, { "id": "8da1c8e54a4a2f939aaf", "model": "gpt-6-luna_max", "organization": "OpenAI", "configuration": "", "score": 0.5609756097560976, "raw_score": 0.5609756097560976, "raw_scale": 1, "source_record": "7n3HBUK684GmdT6Tkycu6m", "evaluated_at": "2026-09-22T22:14:05.000Z" }, { "id": "a39b6746b645b77500e7", "model": "gpt-5.4-2026-03-05_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.49, "raw_score": 0.49, "raw_scale": 1, "source_record": "doiQDghqnLzbE8CFijLNQJ", "evaluated_at": "2026-06-11T17:55:45.000Z" }, { "id": "9fd7e84659a1f2763984", "model": "claude-haiku-5-5_max", "organization": "Anthropic", "configuration": "", "score": 0.4634146341463415, "raw_score": 0.4634146341463415, "raw_scale": 1, "source_record": "WmUXFQoq7GqiSgpTmfLaVv", "evaluated_at": "2026-10-07T20:42:31.000Z" }, { "id": "f8bad9fdd50c6a12a6b8", "model": "deepseek-v4.1-flash-1017_max", "organization": "DeepSeek", "configuration": "", "score": 0.4634146341463415, "raw_score": 0.4634146341463415, "raw_scale": 1, "source_record": "jnCfi6VttQBv3t54B3xp9G", "evaluated_at": "2026-10-19T22:39:00.000Z" }, { "id": "58b2b97eee4082bdda97", "model": "muse-spark-1.3_max", "organization": "Meta AI", "configuration": "", "score": 0.4634146341463415, "raw_score": 0.4634146341463415, "raw_scale": 1, "source_record": "naZno5TPpgMJfv2b5mhvUW", "evaluated_at": "2026-09-18T16:34:54.000Z" }, { "id": "9a8b5148ab6a8bd80866", "model": "qwen3.8-max_xhigh", "organization": "Alibaba", "configuration": "", "score": 0.4634146341463415, "raw_score": 0.4634146341463415, "raw_scale": 1, "source_record": "PBRdMZyz4L4CqPpXyLD2Lk", "evaluated_at": "2026-08-04T19:32:51.000Z" }, { "id": "11ee50f5e621e5204b4e", "model": "gpt-5.2-pro-2025-12-11_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.46, "raw_score": 0.46, "raw_scale": 1, "source_record": "oALBfwNc7dcNSDJSSBJXYe", "evaluated_at": "2026-06-13T02:37:55.000Z" }, { "id": "2ba0df4f3e876dad67c6", "model": "muse-spark-1.3_xhigh", "organization": "Meta AI", "configuration": "", "score": 0.4146341463414634, "raw_score": 0.4146341463414634, "raw_scale": 1, "source_record": "UdTt275jahRApXURferb9i", "evaluated_at": "2026-09-16T02:51:59.000Z" }, { "id": "68ce633e1cc5314c84fc", "model": "kimi-k3_max", "organization": "Moonshot", "configuration": "", "score": 0.3902439024390244, "raw_score": 0.3902439024390244, "raw_scale": 1, "source_record": "gZERSAQwxBAevQzBsWfUie", "evaluated_at": "2026-07-17T01:48:42.000Z" }, { "id": "b01fdb44f8f2ffbb6cf5", "model": "gemini-3.7-flash_high", "organization": "Google DeepMind", "configuration": "", "score": 0.36585365853658536, "raw_score": 0.36585365853658536, "raw_scale": 1, "source_record": "KptKXK5nE2Ns2WbiaSSnbc", "evaluated_at": "2026-08-14T19:59:58.000Z" }, { "id": "a3856a9ed85bbebfadc0", "model": "qwen3.7-max", "organization": "Alibaba", "configuration": "", "score": 0.34146341463414637, "raw_score": 0.34146341463414637, "raw_scale": 1, "source_record": "FHZCKbtfvpKGXU99vZFnXN", "evaluated_at": "2026-06-13T03:08:22.000Z" }, { "id": "4a9b8e60c9e6568c0616", "model": "qwen3.8-max-0902_xhigh", "organization": "Alibaba", "configuration": "", "score": 0.34146341463414637, "raw_score": 0.34146341463414637, "raw_scale": 1, "source_record": "kZUdrqdGtCEJa3ip6L9YnH", "evaluated_at": "2026-09-02T18:04:58.000Z" }, { "id": "deaf55a74b1687d72571", "model": "Muse-Spark-Open-816B-A42B_xhigh", "organization": "Meta AI", "configuration": "", "score": 0.3170731707317073, "raw_score": 0.3170731707317073, "raw_scale": 1, "source_record": "YifYkfNddMSuE8uBNdbvLi", "evaluated_at": "2026-10-15T18:52:04.000Z" }, { "id": "6dc94c339e171cfdb838", "model": "claude-opus-4-7_max", "organization": "Anthropic", "configuration": "", "score": 0.3170731707317073, "raw_score": 0.3170731707317073, "raw_scale": 1, "source_record": "P4LXQ9PyDDziEANMB5jVRd", "evaluated_at": "2026-06-10T21:41:54.000Z" }, { "id": "1b23a0f7ff71a38946ac", "model": "grok-4.6_xhigh", "organization": "xAI", "configuration": "", "score": 0.3170731707317073, "raw_score": 0.3170731707317073, "raw_scale": 1, "source_record": "bg3JRdd33iWfvRoGSvyXXE", "evaluated_at": "2026-08-14T15:46:50.000Z" }, { "id": "c591f5addbd18409ec9e", "model": "gpt-5.2-2025-12-11_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.317, "raw_score": 0.317, "raw_scale": 1, "source_record": "J3NUxsK2GXq9Whkjn7mEVm", "evaluated_at": "2026-06-11T14:48:04.000Z" }, { "id": "78df14271fbe029af503", "model": "claude-sonnet-5_max", "organization": "Anthropic", "configuration": "", "score": 0.2926829268292683, "raw_score": 0.2926829268292683, "raw_scale": 1, "source_record": "bqiVinHXFcgyFGPstJwXJN", "evaluated_at": "2026-06-30T21:54:08.000Z" }, { "id": "10d434e3ece5102beb8b", "model": "glm-5.2_max", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.2926829268292683, "raw_score": 0.2926829268292683, "raw_scale": 1, "source_record": "JebA8dYLVov6H49ZQSLXh5", "evaluated_at": "2026-06-19T16:13:05.000Z" }, { "id": "5a6492ff8f8089478913", "model": "glm-5.3_max", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.2926829268292683, "raw_score": 0.2926829268292683, "raw_scale": 1, "source_record": "4S3ZrJDkwh4gmGrmJQdvzR", "evaluated_at": "2026-08-25T14:20:20.000Z" }, { "id": "fb86ad8a6354c4cc604c", "model": "claude-opus-4-6_max", "organization": "Anthropic", "configuration": "", "score": 0.2682926829268293, "raw_score": 0.2682926829268293, "raw_scale": 1, "source_record": "2hbbp8z6PhB7c2T33os9q5", "evaluated_at": "2026-06-11T00:19:44.000Z" }, { "id": "7352bd4b5e1f44ba1c51", "model": "deepseek-v4-pro-0813_max", "organization": "DeepSeek", "configuration": "", "score": 0.2682926829268293, "raw_score": 0.2682926829268293, "raw_scale": 1, "source_record": "a4uV8HMMBAcHXLVSmZfBgX", "evaluated_at": "2026-08-19T13:35:48.000Z" }, { "id": "2b7ae69147eeb4809ccf", "model": "deepseek-v4.1-flash_max", "organization": "DeepSeek", "configuration": "", "score": 0.2682926829268293, "raw_score": 0.2682926829268293, "raw_scale": 1, "source_record": "KansJcyeGR4m7G4p8GupDn", "evaluated_at": "2026-10-08T19:02:32.000Z" }, { "id": "6ef4553f9a053be7bd94", "model": "gemini-3.1-pro-preview", "organization": "Google DeepMind", "configuration": "", "score": 0.2682926829268293, "raw_score": 0.2682926829268293, "raw_scale": 1, "source_record": "eXVWr3RBeSsWy5DAU8ed46", "evaluated_at": "2026-06-11T23:35:57.000Z" }, { "id": "039342145faf73408502", "model": "gemini-3.5-flash_high", "organization": "Google DeepMind", "configuration": "", "score": 0.2682926829268293, "raw_score": 0.2682926829268293, "raw_scale": 1, "source_record": "cXn5zzns6VFwbTgotDwZjV", "evaluated_at": "2026-06-10T17:32:38.000Z" }, { "id": "a7c19075df6f9cb7e1f8", "model": "mistral-large-4-0_high", "organization": "Mistral AI", "configuration": "", "score": 0.2682926829268293, "raw_score": 0.2682926829268293, "raw_scale": 1, "source_record": "TzoNRjKto6wEuZ5PmQPAw8", "evaluated_at": "2026-10-12T21:38:35.000Z" }, { "id": "7be66603a2c5ee02d6e8", "model": "kimi-k2.6", "organization": "Moonshot", "configuration": "", "score": 0.2564102564102564, "raw_score": 0.2564102564102564, "raw_scale": 1, "source_record": "eVKA2eavdRAbSN6AnzyBxA", "evaluated_at": "2026-06-10T19:47:36.000Z" }, { "id": "455fa6d3cb1dc1be94b6", "model": "deepseek-v4-flash-0731_max", "organization": "DeepSeek", "configuration": "", "score": 0.24390243902439024, "raw_score": 0.24390243902439024, "raw_scale": 1, "source_record": "WYs83xB6NwrkMNyQJtuMcQ", "evaluated_at": "2026-08-02T02:15:37.000Z" }, { "id": "85a3cc6da3d1688ca99d", "model": "grok-4.5_high", "organization": "xAI", "configuration": "", "score": 0.24390243902439024, "raw_score": 0.24390243902439024, "raw_scale": 1, "source_record": "7vThbrtbczScg2UcsAYXtF", "evaluated_at": "2026-07-09T01:53:42.000Z" }, { "id": "84093203cfc1ca85b177", "model": "gemini-3.6-flash_high", "organization": "Google DeepMind", "configuration": "", "score": 0.21951219512195122, "raw_score": 0.21951219512195122, "raw_scale": 1, "source_record": "JCMjXU6FRiiLV3xvs9R2Xv", "evaluated_at": "2026-08-02T00:36:45.000Z" }, { "id": "64d63845a0eef7eadd43", "model": "gemini-3.8-flash_high", "organization": "Google DeepMind", "configuration": "", "score": 0.21951219512195122, "raw_score": 0.21951219512195122, "raw_scale": 1, "source_record": "Lgy52ZT4YpFrUqqKVu2qUs", "evaluated_at": "2026-09-02T17:15:36.000Z" }, { "id": "be6c223b058a186d3bae", "model": "gpt-5-2025-08-07_high", "organization": "OpenAI", "configuration": "", "score": 0.21951219512195122, "raw_score": 0.21951219512195122, "raw_scale": 1, "source_record": "mSS7C84hJyYi4VnJtgzYvW", "evaluated_at": "2026-06-11T01:44:04.000Z" }, { "id": "bb8da9ec803886078ca6", "model": "gpt-5-pro-2025-10-06_high", "organization": "OpenAI", "configuration": "", "score": 0.1951219512195122, "raw_score": 0.1951219512195122, "raw_scale": 1, "source_record": "HfDahF4ZXTN6bjfDFwcHTU", "evaluated_at": "2026-06-12T20:32:51.000Z" }, { "id": "b9dc1b65c214377aac8d", "model": "gemini-3-flash-preview", "organization": "Google DeepMind", "configuration": "", "score": 0.17073170731707318, "raw_score": 0.17073170731707318, "raw_scale": 1, "source_record": "ZczEhu69FQ9Cnns8NT6ew4", "evaluated_at": "2026-06-11T23:44:30.000Z" }, { "id": "bcafdc7842f982669ef5", "model": "glm-5.3-flash_max", "organization": "Z.ai (Zhipu AI)", "configuration": "", "score": 0.17073170731707318, "raw_score": 0.17073170731707318, "raw_scale": 1, "source_record": "iCysWvFJevNyw2dbYosU6J", "evaluated_at": "2026-08-27T13:29:52.000Z" }, { "id": "c2a32ab5679ce1f7d415", "model": "grok-4.20-0309-reasoning", "organization": "xAI", "configuration": "", "score": 0.17073170731707318, "raw_score": 0.17073170731707318, "raw_scale": 1, "source_record": "SX5FCWMk3HF83gGsP3WFA6", "evaluated_at": "2026-07-13T15:51:35.000Z" }, { "id": "f542a315f15bb970521d", "model": "grok-4.7_xhigh", "organization": "xAI", "configuration": "", "score": 0.17073170731707318, "raw_score": 0.17073170731707318, "raw_scale": 1, "source_record": "Ymgj5VeEceBbzsLq6xEZRi", "evaluated_at": "2026-09-22T21:49:22.000Z" }, { "id": "02e6035c4dfd23ca020c", "model": "inkling-small_xhigh", "organization": "Thinking Machines", "configuration": "", "score": 0.17073170731707318, "raw_score": 0.17073170731707318, "raw_scale": 1, "source_record": "DVD8G8DHNyzkowBzK7JvMt", "evaluated_at": "2026-08-14T19:59:09.000Z" }, { "id": "f5a9c9501537d2c86974", "model": "grok-4.3_high", "organization": "xAI", "configuration": "", "score": 0.14634146341463414, "raw_score": 0.14634146341463414, "raw_scale": 1, "source_record": "3iRQkf5v7Wu7YuZhxa76Gg", "evaluated_at": "2026-06-17T17:10:48.000Z" }, { "id": "7f80233e70e9f4914996", "model": "gpt-5-mini-2025-08-07_high", "organization": "OpenAI", "configuration": "", "score": 0.12195121951219512, "raw_score": 0.12195121951219512, "raw_scale": 1, "source_record": "2WxRbjq5ZWTgiXZPqYUG4R", "evaluated_at": "2026-06-12T16:48:15.000Z" }, { "id": "5eeeee2de24184c672cc", "model": "gpt-5.4-nano-2026-03-17_high", "organization": "OpenAI", "configuration": "", "score": 0.12195121951219512, "raw_score": 0.12195121951219512, "raw_scale": 1, "source_record": "SF6FYnnKztPCDWQc4v4EJa", "evaluated_at": "2026-06-12T16:48:06.000Z" }, { "id": "871468a073bb64bff582", "model": "kimi-k2.7-code", "organization": "Moonshot", "configuration": "", "score": 0.12195121951219512, "raw_score": 0.12195121951219512, "raw_scale": 1, "source_record": "UTVSgirxQpTgmTHbgE9EUk", "evaluated_at": "2026-06-13T01:39:44.000Z" }, { "id": "6d15f84baf2fed629111", "model": "gpt-5.4-mini-2026-03-17_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.0975609756097561, "raw_score": 0.0975609756097561, "raw_scale": 1, "source_record": "jtQ6rDiNxjNKkVMEPBE3Ub", "evaluated_at": "2026-06-12T16:40:39.000Z" }, { "id": "50e1ee5f6e1d4b9134f0", "model": "Inkling_xhigh", "organization": "Thinking Machines", "configuration": "", "score": 0.04878048780487805, "raw_score": 0.04878048780487805, "raw_scale": 1, "source_record": "jvJPoRYkPH9Sb5MX252oHp", "evaluated_at": "2026-08-06T20:14:32.000Z" }, { "id": "5e70d05e14b6a6c91236", "model": "claude-opus-4-5-20251101_32K", "organization": "Anthropic", "configuration": "", "score": 0.04878048780487805, "raw_score": 0.04878048780487805, "raw_scale": 1, "source_record": "azk4theXxAsizmoi2HDD49", "evaluated_at": "2026-06-11T01:55:25.000Z" }, { "id": "c764501e178a04e232c7", "model": "o4-mini-2025-04-16_high", "organization": "OpenAI", "configuration": "", "score": 0.04878048780487805, "raw_score": 0.04878048780487805, "raw_scale": 1, "source_record": "TomkQucNryhFGkaUnUGeLR", "evaluated_at": "2026-06-11T13:02:32.385Z" }, { "id": "9c948f0f759cc8772503", "model": "claude-opus-4-1-20250805_32K", "organization": "Anthropic", "configuration": "", "score": 0.024390243902439025, "raw_score": 0.024390243902439025, "raw_scale": 1, "source_record": "9zT72xRR6wZhiwraJTNPyz", "evaluated_at": "2026-06-11T21:15:07.000Z" }, { "id": "3d18b7ac03e9bbf969ac", "model": "claude-sonnet-4-5-20250929_32K", "organization": "Anthropic", "configuration": "", "score": 0.024390243902439025, "raw_score": 0.024390243902439025, "raw_scale": 1, "source_record": "d3m9Xip8cof4s77z6LKkEh", "evaluated_at": "2026-06-11T21:10:25.000Z" }, { "id": "dac1e9eb8fc4fdac88fd", "model": "deepseek-v4-pro_max", "organization": "DeepSeek", "configuration": "", "score": 0.024390243902439025, "raw_score": 0.024390243902439025, "raw_scale": 1, "source_record": "RAZ6VMfGtQSE8EvUZRRz4Q", "evaluated_at": "2026-06-17T02:30:14.000Z" }, { "id": "76480855542d74199796", "model": "gpt-5-nano-2025-08-07_high", "organization": "OpenAI", "configuration": "", "score": 0.024390243902439025, "raw_score": 0.024390243902439025, "raw_scale": 1, "source_record": "P5v3FUnVVTbaqaCr43deKi", "evaluated_at": "2026-06-12T16:47:59.000Z" }, { "id": "40d73b21e99f785ccd12", "model": "gpt-5.5-instant", "organization": "OpenAI", "configuration": "", "score": 0.024390243902439025, "raw_score": 0.024390243902439025, "raw_scale": 1, "source_record": "2UmuBfhBvYbbnTg5BnQqov", "evaluated_at": "2026-08-02T03:23:36.000Z" }, { "id": "ed5ff94898fa62461948", "model": "gemini-2.5-pro", "organization": "Google DeepMind", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "2z7kL9DpfvY44T9Wo99FYx", "evaluated_at": "2026-06-11T23:56:12.000Z" }, { "id": "1b80c8abefb90d918b8c", "model": "gemini-3.5-flash-lite_high", "organization": "Google DeepMind", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "juNVfKBcYTwaKnk9GRA2zL", "evaluated_at": "2026-08-02T01:20:17.000Z" }, { "id": "d784e617e1f5d61b712b", "model": "o3-mini-2025-01-31_high", "organization": "OpenAI", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "iR3Ut7aYEtz8dTYQFzq7rG", "evaluated_at": "2026-06-11T13:04:17.103Z" } ] }, { "id": "mmlu-pro", "name": "MMLU-Pro", "group": "知识与推理", "unit": "%", "metric": "Overall accuracy", "protocol": { "en": "Overall accuracy from rows attributed to TIGER-Lab in the official leaderboard; self-reported submissions are excluded.", "zh": "官方榜单中标注由 TIGER-Lab 测评的 Overall 准确率；不含自行提交的成绩。" }, "source": { "name": "TIGER-Lab", "url": "https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro", "data_url": "https://huggingface.co/datasets/TIGER-Lab/mmlu_pro_leaderboard_submission/resolve/main/results.csv", "retrieved_at": "2026-11-01T16:45:14.196252+00:00", "sha256": "3ba62dcde6605128d0bef50042c23bec8bf420943b678c362ef74e0904cf6db8", "selector": "Data Source=TIGER-Lab (case insensitive); Overall", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "18c7c5baac450bdaf2a0", "model": "Gemini-3.1-Pro", "organization": "Google", "configuration": "", "score": 0.9116, "raw_score": 0.9116, "raw_scale": 1, "source_record": "Gemini-3.1-Pro", "reported_by": "TIGER-Lab" }, { "id": "29e11ffd28bd1cf04b55", "model": "Gemini-2.5-Pro-Exp-03-25", "organization": "Google", "configuration": "", "score": 0.8452, "raw_score": 0.8452, "raw_scale": 1, "source_record": "Gemini-2.5-Pro-Exp-03-25", "reported_by": "TIGER-Lab" }, { "id": "a9b01acfed06b5eef931", "model": "Claude-3.5-Sonnet (2024-10-22)", "organization": "Anthropic", "configuration": "", "score": 0.7764, "raw_score": 0.7764, "raw_scale": 1, "source_record": "Claude-3.5-Sonnet (2024-10-22)", "reported_by": "TIGER-Lab" }, { "id": "59cf098a6bb692cafb4a", "model": "Gemini-2.0-Flash-exp", "organization": "Google", "configuration": "", "score": 0.7624, "raw_score": 0.7624, "raw_scale": 1, "source_record": "Gemini-2.0-Flash-exp", "reported_by": "TIGER-Lab" }, { "id": "62f8acbb38712996bd2f", "model": "Claude-3.5-Sonnet (2024-06-20)", "organization": "Anthropic", "configuration": "", "score": 0.7612, "raw_score": 0.7612, "raw_scale": 1, "source_record": "Claude-3.5-Sonnet (2024-06-20)", "reported_by": "TIGER-Lab" }, { "id": "7c24654ef43cb7d2fba3", "model": "GPT-4o (2024-08-06)", "organization": "OpenAI", "configuration": "", "score": 0.7468, "raw_score": 0.7468, "raw_scale": 1, "source_record": "GPT-4o (2024-08-06)", "reported_by": "TIGER-Lab" }, { "id": "59574ada4e1d1d34a4f2", "model": "GPT-oss-20B(medium)", "organization": "OpenAI", "configuration": "", "score": 0.7314, "raw_score": 0.7314, "raw_scale": 1, "source_record": "GPT-oss-20B(medium)", "reported_by": "TIGER-Lab" }, { "id": "395c3fc36da18ae36e5a", "model": "Athene-V2-Chat (0-shot)", "organization": "", "configuration": "", "score": 0.7311, "raw_score": 0.7311, "raw_scale": 1, "source_record": "Athene-V2-Chat (0-shot)", "reported_by": "TIGER-Lab" }, { "id": "79e5e3b63ab8a829f993", "model": "GPT-4o (2024-05-13)", "organization": "OpenAI", "configuration": "", "score": 0.7255, "raw_score": 0.7255, "raw_scale": 1, "source_record": "GPT-4o (2024-05-13)", "reported_by": "TIGER-Lab" }, { "id": "252556febaf908e90bd9", "model": "Gemini-1.5-Pro-002", "organization": "Google", "configuration": "", "score": 0.7025, "raw_score": 0.7025, "raw_scale": 1, "source_record": "Gemini-1.5-Pro-002", "reported_by": "TIGER-Lab" }, { "id": "737aa6ae38a6f6aca30a", "model": "Athene-V2-Chat", "organization": "", "configuration": "", "score": 0.7021, "raw_score": 0.7021, "raw_scale": 1, "source_record": "Athene-V2-Chat", "reported_by": "TIGER-Lab" }, { "id": "4ff8980a69dd2fb9d733", "model": "QwQ-32B", "organization": "", "configuration": "", "score": 0.6907, "raw_score": 0.6907, "raw_scale": 1, "source_record": "QwQ-32B", "reported_by": "TIGER-Lab" }, { "id": "eee31c8b858349d0ae74", "model": "Claude-3-Opus", "organization": "Anthropic", "configuration": "", "score": 0.6845, "raw_score": 0.6845, "raw_scale": 1, "source_record": "Claude-3-Opus", "reported_by": "TIGER-Lab" }, { "id": "f26ca9823de2b58b5db2", "model": "Mistral-Large-Instruct-2411", "organization": "Mistral AI", "configuration": "", "score": 0.6794, "raw_score": 0.6794, "raw_scale": 1, "source_record": "Mistral-Large-Instruct-2411", "reported_by": "TIGER-Lab" }, { "id": "97d06434c085ed09497b", "model": "Llama-3.3-70B-Instruct", "organization": "Meta", "configuration": "", "score": 0.6592, "raw_score": 0.6592, "raw_scale": 1, "source_record": "Llama-3.3-70B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "d9be016a0671b4df8089", "model": "Mistral-Large-Instruct-2407", "organization": "Mistral AI", "configuration": "", "score": 0.6591, "raw_score": 0.6591, "raw_scale": 1, "source_record": "Mistral-Large-Instruct-2407", "reported_by": "TIGER-Lab" }, { "id": "f206703c47423ff2797f", "model": "DeepSeek-Chat-V2_5", "organization": "DeepSeek", "configuration": "", "score": 0.6583, "raw_score": 0.6583, "raw_scale": 1, "source_record": "DeepSeek-Chat-V2_5", "reported_by": "TIGER-Lab" }, { "id": "ce163386751f026ab2c1", "model": "Qwen2-72B-Chat", "organization": "Alibaba", "configuration": "", "score": 0.6438, "raw_score": 0.6438, "raw_scale": 1, "source_record": "Qwen2-72B-Chat", "reported_by": "TIGER-Lab" }, { "id": "c591599104fb74abac45", "model": "Gemini-1.5-Flash-002", "organization": "Google", "configuration": "", "score": 0.6409, "raw_score": 0.6409, "raw_scale": 1, "source_record": "Gemini-1.5-Flash-002", "reported_by": "TIGER-Lab" }, { "id": "1ed945663c2615c1d37d", "model": "magnum-72b-v1", "organization": "", "configuration": "", "score": 0.6393, "raw_score": 0.6393, "raw_scale": 1, "source_record": "magnum-72b-v1", "reported_by": "TIGER-Lab" }, { "id": "c4dcac7505232e03b7b9", "model": "GPT-4-Turbo", "organization": "OpenAI", "configuration": "", "score": 0.6371, "raw_score": 0.6371, "raw_scale": 1, "source_record": "GPT-4-Turbo", "reported_by": "TIGER-Lab" }, { "id": "531cb4f68416b94fca45", "model": "DeepSeek-Coder-V2-Instruct", "organization": "DeepSeek", "configuration": "", "score": 0.6363, "raw_score": 0.6363, "raw_scale": 1, "source_record": "DeepSeek-Coder-V2-Instruct", "reported_by": "TIGER-Lab" }, { "id": "52c12a62a94732db9384", "model": "GPT-4o-mini", "organization": "OpenAI", "configuration": "", "score": 0.6309, "raw_score": 0.6309, "raw_scale": 1, "source_record": "GPT-4o-mini", "reported_by": "TIGER-Lab" }, { "id": "55daabb935f5211c65ce", "model": "Llama-3.1-70B-Instruct", "organization": "Meta", "configuration": "", "score": 0.6284, "raw_score": 0.6284, "raw_scale": 1, "source_record": "Llama-3.1-70B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "f773f87f4c3772c18e15", "model": "Llama-3.1-Nemotron-70B-Instruct-HF", "organization": "Meta", "configuration": "", "score": 0.6278, "raw_score": 0.6278, "raw_scale": 1, "source_record": "Llama-3.1-Nemotron-70B-Instruct-HF", "reported_by": "TIGER-Lab" }, { "id": "89cc7f5e8489b0c0872c", "model": "Yi-Lightning", "organization": "01.AI", "configuration": "", "score": 0.6238, "raw_score": 0.6238, "raw_scale": 1, "source_record": "Yi-Lightning", "reported_by": "TIGER-Lab" }, { "id": "903ef116da6a758bc605", "model": "Claude-3-5-Haiku-20241022", "organization": "Anthropic", "configuration": "", "score": 0.6212, "raw_score": 0.6212, "raw_scale": 1, "source_record": "Claude-3-5-Haiku-20241022", "reported_by": "TIGER-Lab" }, { "id": "2e6585067083457e57a3", "model": "Reflection-Llama-3.1-70B", "organization": "", "configuration": "", "score": 0.6035, "raw_score": 0.6035, "raw_scale": 1, "source_record": "Reflection-Llama-3.1-70B", "reported_by": "TIGER-Lab" }, { "id": "97effa48de434338558b", "model": "Gemini-1.5-Flash", "organization": "Google", "configuration": "", "score": 0.5912, "raw_score": 0.5912, "raw_scale": 1, "source_record": "Gemini-1.5-Flash", "reported_by": "TIGER-Lab" }, { "id": "c8bcf045f4f526909a4f", "model": "EXAONE-3.5-32B-Instruct", "organization": "", "configuration": "", "score": 0.5891, "raw_score": 0.5891, "raw_scale": 1, "source_record": "EXAONE-3.5-32B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "db500ff86b473b3ae959", "model": "Yi-large", "organization": "01.AI", "configuration": "", "score": 0.5809, "raw_score": 0.5809, "raw_scale": 1, "source_record": "Yi-large", "reported_by": "TIGER-Lab" }, { "id": "6eb91afc2b62b6f7ac11", "model": "Claude-3-Sonnet", "organization": "Anthropic", "configuration": "", "score": 0.568, "raw_score": 0.568, "raw_scale": 1, "source_record": "Claude-3-Sonnet", "reported_by": "TIGER-Lab" }, { "id": "5883aebe1668fa728a3f", "model": "Gemma-2-27B-it", "organization": "Google", "configuration": "", "score": 0.5654, "raw_score": 0.5654, "raw_scale": 1, "source_record": "Gemma-2-27B-it", "reported_by": "TIGER-Lab" }, { "id": "f7e707367b35c8316fa3", "model": "Mixtral-8x22B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "", "score": 0.5633, "raw_score": 0.5633, "raw_scale": 1, "source_record": "Mixtral-8x22B-Instruct-v0.1", "reported_by": "TIGER-Lab" }, { "id": "191e36d7c9e59ab973b8", "model": "Llama-3-70B-Instruct", "organization": "Meta", "configuration": "", "score": 0.562, "raw_score": 0.562, "raw_scale": 1, "source_record": "Llama-3-70B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "025327e567c21339add6", "model": "Phi3-medium-4k", "organization": "", "configuration": "", "score": 0.557, "raw_score": 0.557, "raw_scale": 1, "source_record": "Phi3-medium-4k", "reported_by": "TIGER-Lab" }, { "id": "4ef58c9a7850cf0f87c5", "model": "Qwen2-72B-32k", "organization": "Alibaba", "configuration": "", "score": 0.5559, "raw_score": 0.5559, "raw_scale": 1, "source_record": "Qwen2-72B-32k", "reported_by": "TIGER-Lab" }, { "id": "5fff107d73fb069218e4", "model": "Deepseek-V2-Chat", "organization": "DeepSeek", "configuration": "", "score": 0.5481, "raw_score": 0.5481, "raw_scale": 1, "source_record": "Deepseek-V2-Chat", "reported_by": "TIGER-Lab" }, { "id": "31d16f99a21f0bd414a9", "model": "Llama-3-70B", "organization": "Meta", "configuration": "", "score": 0.5278, "raw_score": 0.5278, "raw_scale": 1, "source_record": "Llama-3-70B", "reported_by": "TIGER-Lab" }, { "id": "4afdad5104fa1b40e87e", "model": "Qwen1.5-72B-Chat", "organization": "Alibaba", "configuration": "", "score": 0.5264, "raw_score": 0.5264, "raw_scale": 1, "source_record": "Qwen1.5-72B-Chat", "reported_by": "TIGER-Lab" }, { "id": "34feef7f51cf10444038", "model": "Llama-3.1-70B", "organization": "Meta", "configuration": "", "score": 0.5247, "raw_score": 0.5247, "raw_scale": 1, "source_record": "Llama-3.1-70B", "reported_by": "TIGER-Lab" }, { "id": "65ef3b93e8c26ddc5013", "model": "Yi-1.5-34B-Chat", "organization": "01.AI", "configuration": "", "score": 0.5229, "raw_score": 0.5229, "raw_scale": 1, "source_record": "Yi-1.5-34B-Chat", "reported_by": "TIGER-Lab" }, { "id": "984d9e1e0ccf75bf87ed", "model": "Gemma-2-9B-it", "organization": "Google", "configuration": "", "score": 0.5208, "raw_score": 0.5208, "raw_scale": 1, "source_record": "Gemma-2-9B-it", "reported_by": "TIGER-Lab" }, { "id": "19b966c7037b7312f4f1", "model": "Phi3-medium-128k", "organization": "", "configuration": "", "score": 0.5191, "raw_score": 0.5191, "raw_scale": 1, "source_record": "Phi3-medium-128k", "reported_by": "TIGER-Lab" }, { "id": "adff7cfde0dc8e89b9f9", "model": "MAmmoTH2-8x7B-Plus", "organization": "", "configuration": "", "score": 0.504, "raw_score": 0.504, "raw_scale": 1, "source_record": "MAmmoTH2-8x7B-Plus", "reported_by": "TIGER-Lab" }, { "id": "5ca471929a2b7568e722", "model": "Qwen1.5-110B", "organization": "Alibaba", "configuration": "", "score": 0.4993, "raw_score": 0.4993, "raw_scale": 1, "source_record": "Qwen1.5-110B", "reported_by": "TIGER-Lab" }, { "id": "5d02768b9d68a9e4709a", "model": "Jamba-1.5-Large", "organization": "", "configuration": "", "score": 0.4946, "raw_score": 0.4946, "raw_scale": 1, "source_record": "Jamba-1.5-Large", "reported_by": "TIGER-Lab" }, { "id": "dd07a3eb0ca36bde1336", "model": "Mistral-Small-Instruct-2409", "organization": "Mistral AI", "configuration": "", "score": 0.484, "raw_score": 0.484, "raw_scale": 1, "source_record": "Mistral-Small-Instruct-2409", "reported_by": "TIGER-Lab" }, { "id": "876bfc11abdf7d06cd0a", "model": "GLM-4-9B-Chat", "organization": "Z AI", "configuration": "", "score": 0.4801, "raw_score": 0.4801, "raw_scale": 1, "source_record": "GLM-4-9B-Chat", "reported_by": "TIGER-Lab" }, { "id": "c8674f38586c555544ba", "model": "GLM-4-9B", "organization": "Z AI", "configuration": "", "score": 0.4792, "raw_score": 0.4792, "raw_scale": 1, "source_record": "GLM-4-9B", "reported_by": "TIGER-Lab" }, { "id": "4044097005df213567c2", "model": "Phi-3.5-mini-instruct", "organization": "Microsoft", "configuration": "", "score": 0.4787, "raw_score": 0.4787, "raw_scale": 1, "source_record": "Phi-3.5-mini-instruct", "reported_by": "TIGER-Lab" }, { "id": "e6e26b4b9e2582b38680", "model": "Qwen2-7B-Instruct", "organization": "Alibaba", "configuration": "", "score": 0.4724, "raw_score": 0.4724, "raw_scale": 1, "source_record": "Qwen2-7B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "18e34672ab627298d24d", "model": "EXAONE-3.5-7.8B-Instruct", "organization": "", "configuration": "", "score": 0.4624, "raw_score": 0.4624, "raw_scale": 1, "source_record": "EXAONE-3.5-7.8B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "789765d27879537523d7", "model": "Yi-1.5-9B-Chat", "organization": "01.AI", "configuration": "", "score": 0.4595, "raw_score": 0.4595, "raw_scale": 1, "source_record": "Yi-1.5-9B-Chat", "reported_by": "TIGER-Lab" }, { "id": "04d4c95d3ed3897e455e", "model": "Phi3-mini-4k", "organization": "", "configuration": "", "score": 0.4566, "raw_score": 0.4566, "raw_scale": 1, "source_record": "Phi3-mini-4k", "reported_by": "TIGER-Lab" }, { "id": "ca5af62460ed8be85751", "model": "Aya-Expanse-32B", "organization": "", "configuration": "", "score": 0.4541, "raw_score": 0.4541, "raw_scale": 1, "source_record": "Aya-Expanse-32B", "reported_by": "TIGER-Lab" }, { "id": "56de3dd0da9d8f8b6858", "model": "Gemma-2-9B", "organization": "Google", "configuration": "", "score": 0.451, "raw_score": 0.451, "raw_scale": 1, "source_record": "Gemma-2-9B", "reported_by": "TIGER-Lab" }, { "id": "340c68ea4e0d3455763d", "model": "Mistral-Nemo-Instruct-2407", "organization": "Mistral AI", "configuration": "", "score": 0.4481, "raw_score": 0.4481, "raw_scale": 1, "source_record": "Mistral-Nemo-Instruct-2407", "reported_by": "TIGER-Lab" }, { "id": "f9b45bd93750ea1d1681", "model": "Llama-3.1-8B-Instruct", "organization": "Meta", "configuration": "", "score": 0.4425, "raw_score": 0.4425, "raw_scale": 1, "source_record": "Llama-3.1-8B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "e5f5ab2375124b87ada0", "model": "Phi3-mini-128k", "organization": "", "configuration": "", "score": 0.4386, "raw_score": 0.4386, "raw_scale": 1, "source_record": "Phi3-mini-128k", "reported_by": "TIGER-Lab" }, { "id": "f68cbac20ecd6fb0fb33", "model": "MAmmoTH2-8B-Plus", "organization": "", "configuration": "", "score": 0.4335, "raw_score": 0.4335, "raw_scale": 1, "source_record": "MAmmoTH2-8B-Plus", "reported_by": "TIGER-Lab" }, { "id": "0a800dac99fb938f31f2", "model": "Mixtral-8x7B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "", "score": 0.4327, "raw_score": 0.4327, "raw_scale": 1, "source_record": "Mixtral-8x7B-Instruct-v0.1", "reported_by": "TIGER-Lab" }, { "id": "419b5a79160e8b0fb7ac", "model": "Yi-34B", "organization": "01.AI", "configuration": "", "score": 0.4303, "raw_score": 0.4303, "raw_scale": 1, "source_record": "Yi-34B", "reported_by": "TIGER-Lab" }, { "id": "82ee262512a54a309c96", "model": "Claude-3-Haiku-20240307", "organization": "Anthropic", "configuration": "", "score": 0.4229, "raw_score": 0.4229, "raw_scale": 1, "source_record": "Claude-3-Haiku-20240307", "reported_by": "TIGER-Lab" }, { "id": "832a959ac588e5926493", "model": "Mathstral-7B-v0.1", "organization": "", "configuration": "", "score": 0.42, "raw_score": 0.42, "raw_scale": 1, "source_record": "Mathstral-7B-v0.1", "reported_by": "TIGER-Lab" }, { "id": "25841a38654c0d8cc1fa", "model": "DeepSeek-Coder-V2-Lite-Instruct", "organization": "DeepSeek", "configuration": "", "score": 0.4157, "raw_score": 0.4157, "raw_scale": 1, "source_record": "DeepSeek-Coder-V2-Lite-Instruct", "reported_by": "TIGER-Lab" }, { "id": "fdf985af3952582c2020", "model": "Granite-3.1-8B-Instruct", "organization": "", "configuration": "", "score": 0.4103, "raw_score": 0.4103, "raw_scale": 1, "source_record": "Granite-3.1-8B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "231343eb41cc60b5398c", "model": "Mixtral-8x7B-v0.1", "organization": "Mistral AI", "configuration": "", "score": 0.4103, "raw_score": 0.4103, "raw_scale": 1, "source_record": "Mixtral-8x7B-v0.1", "reported_by": "TIGER-Lab" }, { "id": "cce44d106c4b3990eee1", "model": "Llama-3-8B-Instruct", "organization": "Meta", "configuration": "", "score": 0.4098, "raw_score": 0.4098, "raw_scale": 1, "source_record": "Llama-3-8B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "d93b1e0277bb1f0ee881", "model": "MAmmoTH2-7B-Plus", "organization": "", "configuration": "", "score": 0.4085, "raw_score": 0.4085, "raw_scale": 1, "source_record": "MAmmoTH2-7B-Plus", "reported_by": "TIGER-Lab" }, { "id": "fa9efbe1efec5d0b0d38", "model": "Qwen2-7B", "organization": "Alibaba", "configuration": "", "score": 0.4073, "raw_score": 0.4073, "raw_scale": 1, "source_record": "Qwen2-7B", "reported_by": "TIGER-Lab" }, { "id": "3806915f6a02f5e0b88c", "model": "Mistral-Nemo-Base-2407", "organization": "Mistral AI", "configuration": "", "score": 0.3977, "raw_score": 0.3977, "raw_scale": 1, "source_record": "Mistral-Nemo-Base-2407", "reported_by": "TIGER-Lab" }, { "id": "fcae66ae7bd4f4754292", "model": "WizardLM-2-8x22B", "organization": "", "configuration": "", "score": 0.3924, "raw_score": 0.3924, "raw_scale": 1, "source_record": "WizardLM-2-8x22B", "reported_by": "TIGER-Lab" }, { "id": "81ee9949fdd8902c9cb4", "model": "EXAONE-3.5-2.4B-Instruct", "organization": "", "configuration": "", "score": 0.391, "raw_score": 0.391, "raw_scale": 1, "source_record": "EXAONE-3.5-2.4B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "1cf7cd4c4f5fe21df135", "model": "Yi-1.5-6B-Chat", "organization": "01.AI", "configuration": "", "score": 0.3823, "raw_score": 0.3823, "raw_scale": 1, "source_record": "Yi-1.5-6B-Chat", "reported_by": "TIGER-Lab" }, { "id": "467cddfe6f8f2f2d14a0", "model": "Qwen1.5-14B-Chat", "organization": "Alibaba", "configuration": "", "score": 0.3802, "raw_score": 0.3802, "raw_scale": 1, "source_record": "Qwen1.5-14B-Chat", "reported_by": "TIGER-Lab" }, { "id": "2a3381ebb7233e0e6a5d", "model": "Ministral-8B-Instruct-2410", "organization": "", "configuration": "", "score": 0.3793, "raw_score": 0.3793, "raw_scale": 1, "source_record": "Ministral-8B-Instruct-2410", "reported_by": "TIGER-Lab" }, { "id": "5293c8f62634686eacea", "model": "Staring-7B", "organization": "", "configuration": "", "score": 0.379, "raw_score": 0.379, "raw_scale": 1, "source_record": "Staring-7B", "reported_by": "TIGER-Lab" }, { "id": "13a1cab4e22ad57485fd", "model": "c4ai-command-r-v01", "organization": "", "configuration": "", "score": 0.379, "raw_score": 0.379, "raw_scale": 1, "source_record": "c4ai-command-r-v01", "reported_by": "TIGER-Lab" }, { "id": "960c89f3137d678f14ef", "model": "Llama-2-70B", "organization": "Meta", "configuration": "", "score": 0.3753, "raw_score": 0.3753, "raw_scale": 1, "source_record": "Llama-2-70B", "reported_by": "TIGER-Lab" }, { "id": "e4d24932e6abd198e372", "model": "OpenChat-3.5-8B", "organization": "", "configuration": "", "score": 0.3724, "raw_score": 0.3724, "raw_scale": 1, "source_record": "OpenChat-3.5-8B", "reported_by": "TIGER-Lab" }, { "id": "33f44ce40841fcd30ece", "model": "InternMath-20B-Plus", "organization": "", "configuration": "", "score": 0.371, "raw_score": 0.371, "raw_scale": 1, "source_record": "InternMath-20B-Plus", "reported_by": "TIGER-Lab" }, { "id": "17cc9121b794b8b0ca17", "model": "Llama3-Smaug-8B", "organization": "Meta", "configuration": "", "score": 0.3693, "raw_score": 0.3693, "raw_scale": 1, "source_record": "Llama3-Smaug-8B", "reported_by": "TIGER-Lab" }, { "id": "252b3fc914cd17d8d76e", "model": "Llama-3.1-8B", "organization": "Meta", "configuration": "", "score": 0.366, "raw_score": 0.366, "raw_scale": 1, "source_record": "Llama-3.1-8B", "reported_by": "TIGER-Lab" }, { "id": "f150cf1e3f4e66635a62", "model": "Llama-3-8B", "organization": "Meta", "configuration": "", "score": 0.3536, "raw_score": 0.3536, "raw_scale": 1, "source_record": "Llama-3-8B", "reported_by": "TIGER-Lab" }, { "id": "21eca255ee81512dc952", "model": "DeepseekMath-7B-Instruct", "organization": "DeepSeek", "configuration": "", "score": 0.353, "raw_score": 0.353, "raw_scale": 1, "source_record": "DeepseekMath-7B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "7603f5344a4b9ae2ddec", "model": "DeepSeek-Coder-V2-Lite-Base", "organization": "DeepSeek", "configuration": "", "score": 0.3437, "raw_score": 0.3437, "raw_scale": 1, "source_record": "DeepSeek-Coder-V2-Lite-Base", "reported_by": "TIGER-Lab" }, { "id": "4a2a3682e85beb93c517", "model": "Aya-Expanse-8B", "organization": "", "configuration": "", "score": 0.3374, "raw_score": 0.3374, "raw_scale": 1, "source_record": "Aya-Expanse-8B", "reported_by": "TIGER-Lab" }, { "id": "89e086af5288bcd64952", "model": "Gemma-7B", "organization": "Google", "configuration": "", "score": 0.3373, "raw_score": 0.3373, "raw_scale": 1, "source_record": "Gemma-7B", "reported_by": "TIGER-Lab" }, { "id": "116af16faa5ac27e248e", "model": "InternMath-7B-Plus", "organization": "", "configuration": "", "score": 0.335, "raw_score": 0.335, "raw_scale": 1, "source_record": "InternMath-7B-Plus", "reported_by": "TIGER-Lab" }, { "id": "f7f3b97810eedfeb13f5", "model": "Granite-3.1-8B-Base", "organization": "", "configuration": "", "score": 0.3308, "raw_score": 0.3308, "raw_scale": 1, "source_record": "Granite-3.1-8B-Base", "reported_by": "TIGER-Lab" }, { "id": "74a752d2e8c02aef1dcc", "model": "Zephyr-7B-Beta", "organization": "", "configuration": "", "score": 0.3297, "raw_score": 0.3297, "raw_scale": 1, "source_record": "Zephyr-7B-Beta", "reported_by": "TIGER-Lab" }, { "id": "ffc5f111378336b0f502", "model": "Granite-3.1-2B-Instruct", "organization": "", "configuration": "", "score": 0.3197, "raw_score": 0.3197, "raw_scale": 1, "source_record": "Granite-3.1-2B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "7b94220c6abb8fb9ce50", "model": "Granite-3.0-8B-Base", "organization": "", "configuration": "", "score": 0.3103, "raw_score": 0.3103, "raw_scale": 1, "source_record": "Granite-3.0-8B-Base", "reported_by": "TIGER-Lab" }, { "id": "b5119b816f7c278ec433", "model": "Mistral-7B-v0.1", "organization": "Mistral AI", "configuration": "", "score": 0.3088, "raw_score": 0.3088, "raw_scale": 1, "source_record": "Mistral-7B-v0.1", "reported_by": "TIGER-Lab" }, { "id": "c55a8bf0730a3573e29d", "model": "Mistral-7B-Instruct-v0.2", "organization": "Mistral AI", "configuration": "", "score": 0.3084, "raw_score": 0.3084, "raw_scale": 1, "source_record": "Mistral-7B-Instruct-v0.2", "reported_by": "TIGER-Lab" }, { "id": "5f8c5d42c6681e023f38", "model": "Mistral-7B-v0.2", "organization": "Mistral AI", "configuration": "", "score": 0.3043, "raw_score": 0.3043, "raw_scale": 1, "source_record": "Mistral-7B-v0.2", "reported_by": "TIGER-Lab" }, { "id": "8c8eb57850b20ff1e443", "model": "Qwen1.5-7B-Chat", "organization": "Alibaba", "configuration": "", "score": 0.2906, "raw_score": 0.2906, "raw_scale": 1, "source_record": "Qwen1.5-7B-Chat", "reported_by": "TIGER-Lab" }, { "id": "c9ee506cf94f569b0a30", "model": "Yi-6B-Chat", "organization": "01.AI", "configuration": "", "score": 0.2884, "raw_score": 0.2884, "raw_scale": 1, "source_record": "Yi-6B-Chat", "reported_by": "TIGER-Lab" }, { "id": "19aaf6e2ec84e2343586", "model": "Neo-7B-Instruct", "organization": "", "configuration": "", "score": 0.2874, "raw_score": 0.2874, "raw_scale": 1, "source_record": "Neo-7B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "fbf3f2eb538891352a5e", "model": "Yi-6B", "organization": "01.AI", "configuration": "", "score": 0.2651, "raw_score": 0.2651, "raw_scale": 1, "source_record": "Yi-6B", "reported_by": "TIGER-Lab" }, { "id": "89fd9780cf4d5f0dce63", "model": "Neo-7B", "organization": "", "configuration": "", "score": 0.2585, "raw_score": 0.2585, "raw_scale": 1, "source_record": "Neo-7B", "reported_by": "TIGER-Lab" }, { "id": "422861d3fbd8cf9ee4c5", "model": "Mistral-7B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "", "score": 0.2575, "raw_score": 0.2575, "raw_scale": 1, "source_record": "Mistral-7B-Instruct-v0.1", "reported_by": "TIGER-Lab" }, { "id": "768ecef3bebc156934d6", "model": "Granite-3.1-3B-A800M-Instruct", "organization": "", "configuration": "", "score": 0.2542, "raw_score": 0.2542, "raw_scale": 1, "source_record": "Granite-3.1-3B-A800M-Instruct", "reported_by": "TIGER-Lab" }, { "id": "ecdddf651f27da219ad2", "model": "Llama-2-13B", "organization": "Meta", "configuration": "", "score": 0.2534, "raw_score": 0.2534, "raw_scale": 1, "source_record": "Llama-2-13B", "reported_by": "TIGER-Lab" }, { "id": "d5abd5c76ea39043aef1", "model": "Granite-3.1-2B-Base", "organization": "", "configuration": "", "score": 0.2389, "raw_score": 0.2389, "raw_scale": 1, "source_record": "Granite-3.1-2B-Base", "reported_by": "TIGER-Lab" }, { "id": "27a74b8cf836a414254a", "model": "Llemma-7B", "organization": "", "configuration": "", "score": 0.2345, "raw_score": 0.2345, "raw_scale": 1, "source_record": "Llemma-7B", "reported_by": "TIGER-Lab" }, { "id": "ddec5b5741a6600e5847", "model": "Qwen2-1.5B-Instruct", "organization": "Alibaba", "configuration": "", "score": 0.2262, "raw_score": 0.2262, "raw_scale": 1, "source_record": "Qwen2-1.5B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "2a5e0d026853d5c02f60", "model": "Qwen2-1.5B", "organization": "Alibaba", "configuration": "", "score": 0.2256, "raw_score": 0.2256, "raw_scale": 1, "source_record": "Qwen2-1.5B", "reported_by": "TIGER-Lab" }, { "id": "961cc39e1bb26fedd7a4", "model": "Llama-3.2-3B", "organization": "Meta", "configuration": "", "score": 0.2217, "raw_score": 0.2217, "raw_scale": 1, "source_record": "Llama-3.2-3B", "reported_by": "TIGER-Lab" }, { "id": "329716f766b376890dc9", "model": "Granite-3.0-2B-Base", "organization": "", "configuration": "", "score": 0.2172, "raw_score": 0.2172, "raw_scale": 1, "source_record": "Granite-3.0-2B-Base", "reported_by": "TIGER-Lab" }, { "id": "e5e411d27bd93de9e854", "model": "Granite-3.1-3B-A800M-Base", "organization": "", "configuration": "", "score": 0.2039, "raw_score": 0.2039, "raw_scale": 1, "source_record": "Granite-3.1-3B-A800M-Base", "reported_by": "TIGER-Lab" }, { "id": "def3434fc76ab3549438", "model": "Llama-2-7B", "organization": "Meta", "configuration": "", "score": 0.2032, "raw_score": 0.2032, "raw_scale": 1, "source_record": "Llama-2-7B", "reported_by": "TIGER-Lab" }, { "id": "6f95bbc44e0f0a965b49", "model": "SmolLM2-1.7B", "organization": "", "configuration": "", "score": 0.1831, "raw_score": 0.1831, "raw_scale": 1, "source_record": "SmolLM2-1.7B", "reported_by": "TIGER-Lab" }, { "id": "74c23dc3986153ca8a4e", "model": "Qwen2-0.5B-Instruct", "organization": "Alibaba", "configuration": "", "score": 0.1593, "raw_score": 0.1593, "raw_scale": 1, "source_record": "Qwen2-0.5B-Instruct", "reported_by": "TIGER-Lab" }, { "id": "219a51749bd51d6dc1e6", "model": "Gemma-2B", "organization": "Google", "configuration": "", "score": 0.1585, "raw_score": 0.1585, "raw_scale": 1, "source_record": "Gemma-2B", "reported_by": "TIGER-Lab" }, { "id": "ccdea390014d50c8cc86", "model": "Qwen2-0.5B", "organization": "Alibaba", "configuration": "", "score": 0.1497, "raw_score": 0.1497, "raw_scale": 1, "source_record": "Qwen2-0.5B", "reported_by": "TIGER-Lab" }, { "id": "6488db0f56ea0229545e", "model": "Granite-3.1-1B-A400M-Instruct", "organization": "", "configuration": "", "score": 0.1327, "raw_score": 0.1327, "raw_scale": 1, "source_record": "Granite-3.1-1B-A400M-Instruct", "reported_by": "TIGER-Lab" }, { "id": "fb01a7f78a582dbe4b7f", "model": "Granite-3.1-1B-A400M-Base", "organization": "", "configuration": "", "score": 0.1234, "raw_score": 0.1234, "raw_scale": 1, "source_record": "Granite-3.1-1B-A400M-Base", "reported_by": "TIGER-Lab" }, { "id": "e0aff3845605cbcc78f3", "model": "Llama-3.2-1B", "organization": "Meta", "configuration": "", "score": 0.1195, "raw_score": 0.1195, "raw_scale": 1, "source_record": "Llama-3.2-1B", "reported_by": "TIGER-Lab" }, { "id": "977550bba23899819158", "model": "SmolLM-1.7B", "organization": "", "configuration": "", "score": 0.1193, "raw_score": 0.1193, "raw_scale": 1, "source_record": "SmolLM-1.7B", "reported_by": "TIGER-Lab" }, { "id": "e9a34330c8a203846f48", "model": "SmolLM2-360M", "organization": "", "configuration": "", "score": 0.1138, "raw_score": 0.1138, "raw_scale": 1, "source_record": "SmolLM2-360M", "reported_by": "TIGER-Lab" }, { "id": "867829e922f7fcbeb7c3", "model": "SmolLM-135M", "organization": "", "configuration": "", "score": 0.1122, "raw_score": 0.1122, "raw_scale": 1, "source_record": "SmolLM-135M", "reported_by": "TIGER-Lab" }, { "id": "ac2aac4b0588598689e1", "model": "SmolLM-360M", "organization": "", "configuration": "", "score": 0.1095, "raw_score": 0.1095, "raw_scale": 1, "source_record": "SmolLM-360M", "reported_by": "TIGER-Lab" }, { "id": "64d3e3fae1fb9932aa35", "model": "SmolLM2-135M", "organization": "", "configuration": "", "score": 0.1085, "raw_score": 0.1085, "raw_scale": 1, "source_record": "SmolLM2-135M", "reported_by": "TIGER-Lab" } ] }, { "id": "bfcl-v4", "name": "BFCL v4", "group": "Agent 与工具", "unit": "%", "metric": "Overall accuracy", "protocol": { "en": "BFCL v4 overall accuracy across subcategories. FC and Prompt labels retain the reported calling mode.", "zh": "BFCL v4 各子类汇总准确率；FC 和 Prompt 标签保留原始工具调用方式。" }, "source": { "name": "Berkeley / Gorilla", "url": "https://gorilla.cs.berkeley.edu/leaderboard.html", "data_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", "retrieved_at": "2026-11-01T16:45:15.382038+00:00", "sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "selector": "data_overall.csv: Overall Acc / 100", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "5d7c69231cad4d2ac530", "model": "Claude-Opus-4-5-20251101 (FC)", "organization": "Anthropic", "configuration": "", "score": 0.7746999999999999, "raw_score": 77.47, "raw_scale": 100, "source_record": "Claude-Opus-4-5-20251101 (FC)" }, { "id": "0434954c3b4f7240a16d", "model": "Claude-Sonnet-4-5-20250929 (FC)", "organization": "Anthropic", "configuration": "", "score": 0.7323999999999999, "raw_score": 73.24, "raw_scale": 100, "source_record": "Claude-Sonnet-4-5-20250929 (FC)" }, { "id": "ab4ee586d523f4c3c52e", "model": "Gemini-3-Pro-Preview (Prompt)", "organization": "Google", "configuration": "", "score": 0.7251000000000001, "raw_score": 72.51, "raw_scale": 100, "source_record": "Gemini-3-Pro-Preview (Prompt)" }, { "id": "4ee4224d398d3d1d175e", "model": "GLM-4.6 (FC thinking)", "organization": "Zhipu AI", "configuration": "", "score": 0.7238, "raw_score": 72.38, "raw_scale": 100, "source_record": "GLM-4.6 (FC thinking)" }, { "id": "23277ca889281ad8b20b", "model": "Grok-4-1-fast-reasoning (FC)", "organization": "xAI", "configuration": "", "score": 0.6957, "raw_score": 69.57, "raw_scale": 100, "source_record": "Grok-4-1-fast-reasoning (FC)" }, { "id": "b2316987a9827e6c7bfe", "model": "Claude-Haiku-4-5-20251001 (FC)", "organization": "Anthropic", "configuration": "", "score": 0.687, "raw_score": 68.7, "raw_scale": 100, "source_record": "Claude-Haiku-4-5-20251001 (FC)" }, { "id": "29cbba585dc9672594f1", "model": "Gemini-3-Pro-Preview (FC)", "organization": "Google", "configuration": "", "score": 0.6814, "raw_score": 68.14, "raw_scale": 100, "source_record": "Gemini-3-Pro-Preview (FC)" }, { "id": "20501c78c6af3780e494", "model": "o3-2025-04-16 (Prompt)", "organization": "OpenAI", "configuration": "", "score": 0.6305, "raw_score": 63.05, "raw_scale": 100, "source_record": "o3-2025-04-16 (Prompt)" }, { "id": "2e0e372ba6142e1e5d93", "model": "Grok-4-0709 (Prompt)", "organization": "xAI", "configuration": "", "score": 0.6297, "raw_score": 62.97, "raw_scale": 100, "source_record": "Grok-4-0709 (Prompt)" }, { "id": "543a7d44aeefbd5e04e5", "model": "Grok-4-0709 (FC)", "organization": "xAI", "configuration": "", "score": 0.6138, "raw_score": 61.38, "raw_scale": 100, "source_record": "Grok-4-0709 (FC)" }, { "id": "a93fcc806a4e2a82e9f3", "model": "Moonshotai-Kimi-K2-Instruct (FC)", "organization": "MoonshotAI", "configuration": "", "score": 0.5906, "raw_score": 59.06, "raw_scale": 100, "source_record": "Moonshotai-Kimi-K2-Instruct (FC)" }, { "id": "75bf5370042109da5f9b", "model": "Grok-4-1-fast-non-reasoning (FC)", "organization": "xAI", "configuration": "", "score": 0.5829, "raw_score": 58.29, "raw_scale": 100, "source_record": "Grok-4-1-fast-non-reasoning (FC)" }, { "id": "046ae5a5bdc8a9c1b86f", "model": "Command A Reasoning (FC)", "organization": "Cohere", "configuration": "", "score": 0.5706, "raw_score": 57.06, "raw_scale": 100, "source_record": "Command A Reasoning (FC)" }, { "id": "7bc6fd38823dff952e68", "model": "DeepSeek-V3.2-Exp (Prompt + Thinking)", "organization": "DeepSeek", "configuration": "", "score": 0.5672999999999999, "raw_score": 56.73, "raw_scale": 100, "source_record": "DeepSeek-V3.2-Exp (Prompt + Thinking)" }, { "id": "848edce8756c79c676dc", "model": "Gemini-2.5-Flash (FC)", "organization": "Google", "configuration": "", "score": 0.5624, "raw_score": 56.24, "raw_scale": 100, "source_record": "Gemini-2.5-Flash (FC)" }, { "id": "153bb5e2bec20508bee8", "model": "GPT-5.2-2025-12-11 (FC)", "organization": "OpenAI", "configuration": "", "score": 0.5587, "raw_score": 55.87, "raw_scale": 100, "source_record": "GPT-5.2-2025-12-11 (FC)" }, { "id": "84b117cf979969d48f84", "model": "GPT-5-mini-2025-08-07 (FC)", "organization": "OpenAI", "configuration": "", "score": 0.5546, "raw_score": 55.46, "raw_scale": 100, "source_record": "GPT-5-mini-2025-08-07 (FC)" }, { "id": "fdbe070101f2cdbddeea", "model": "xLAM-2-32b-fc-r (FC)", "organization": "Salesforce", "configuration": "", "score": 0.5466, "raw_score": 54.66, "raw_scale": 100, "source_record": "xLAM-2-32b-fc-r (FC)" }, { "id": "03fdf03470282e2b582b", "model": "DeepSeek-V3.2-Exp (FC)", "organization": "DeepSeek", "configuration": "", "score": 0.5412, "raw_score": 54.12, "raw_scale": 100, "source_record": "DeepSeek-V3.2-Exp (FC)" }, { "id": "89d20511d50649ed38ba", "model": "GPT-4.1-2025-04-14 (FC)", "organization": "OpenAI", "configuration": "", "score": 0.5396, "raw_score": 53.96, "raw_scale": 100, "source_record": "GPT-4.1-2025-04-14 (FC)" }, { "id": "6fd2d2233e02dfea7e88", "model": "o4-mini-2025-04-16 (FC)", "organization": "OpenAI", "configuration": "", "score": 0.5324, "raw_score": 53.24, "raw_scale": 100, "source_record": "o4-mini-2025-04-16 (FC)" }, { "id": "babf30692034265aaf5f", "model": "xLAM-2-70b-fc-r (FC)", "organization": "Salesforce", "configuration": "", "score": 0.5307, "raw_score": 53.07, "raw_scale": 100, "source_record": "xLAM-2-70b-fc-r (FC)" }, { "id": "8f9fdfef03573f3bc380", "model": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", "organization": "Qwen", "configuration": "", "score": 0.5215, "raw_score": 52.15, "raw_scale": 100, "source_record": "Qwen3-235B-A22B-Instruct-2507 (Prompt)" }, { "id": "701380c9fd68248d7497", "model": "GPT-5-nano-2025-08-07 (FC)", "organization": "OpenAI", "configuration": "", "score": 0.5145000000000001, "raw_score": 51.45, "raw_scale": 100, "source_record": "GPT-5-nano-2025-08-07 (FC)" }, { "id": "0038e389b24bbc87818e", "model": "Nanbeige4-3B-Thinking-2511 (FC)", "organization": "Nanbeige", "configuration": "", "score": 0.514, "raw_score": 51.4, "raw_scale": 100, "source_record": "Nanbeige4-3B-Thinking-2511 (FC)" }, { "id": "13f608298c90ca411c5e", "model": "Gemini-2.5-Flash (Prompt)", "organization": "Google", "configuration": "", "score": 0.509, "raw_score": 50.9, "raw_scale": 100, "source_record": "Gemini-2.5-Flash (Prompt)" }, { "id": "359884d4c199a6104f41", "model": "GPT-4.1-mini-2025-04-14 (FC)", "organization": "OpenAI", "configuration": "", "score": 0.5045000000000001, "raw_score": 50.45, "raw_scale": 100, "source_record": "GPT-4.1-mini-2025-04-14 (FC)" }, { "id": "e56e60134cf4c510f085", "model": "o4-mini-2025-04-16 (Prompt)", "organization": "OpenAI", "configuration": "", "score": 0.5025999999999999, "raw_score": 50.26, "raw_scale": 100, "source_record": "o4-mini-2025-04-16 (Prompt)" }, { "id": "3dd11044c4bdedd9b913", "model": "Qwen3-32B (FC)", "organization": "Qwen", "configuration": "", "score": 0.48710000000000003, "raw_score": 48.71, "raw_scale": 100, "source_record": "Qwen3-32B (FC)" }, { "id": "ee2a27d129821c75de89", "model": "o3-2025-04-16 (FC)", "organization": "OpenAI", "configuration": "", "score": 0.48560000000000003, "raw_score": 48.56, "raw_scale": 100, "source_record": "o3-2025-04-16 (FC)" }, { "id": "13fb18ec25cf6b6eee4b", "model": "Qwen3-235B-A22B-Instruct-2507 (FC)", "organization": "Qwen", "configuration": "", "score": 0.4799, "raw_score": 47.99, "raw_scale": 100, "source_record": "Qwen3-235B-A22B-Instruct-2507 (FC)" }, { "id": "eba21a308772afe387c6", "model": "Nanbeige3.5-Pro-Thinking (FC)", "organization": "Nanbeige", "configuration": "", "score": 0.4768, "raw_score": 47.68, "raw_scale": 100, "source_record": "Nanbeige3.5-Pro-Thinking (FC)" }, { "id": "ef8e43899bfd7b7cf330", "model": "Qwen3-32B (Prompt)", "organization": "Qwen", "configuration": "", "score": 0.4678, "raw_score": 46.78, "raw_scale": 100, "source_record": "Qwen3-32B (Prompt)" }, { "id": "b087d3cf80700088c4dc", "model": "xLAM-2-8b-fc-r (FC)", "organization": "Salesforce", "configuration": "", "score": 0.4668, "raw_score": 46.68, "raw_scale": 100, "source_record": "xLAM-2-8b-fc-r (FC)" }, { "id": "a069f6dcbc0e75fb6aa1", "model": "Command A (FC)", "organization": "Cohere", "configuration": "", "score": 0.46490000000000004, "raw_score": 46.49, "raw_scale": 100, "source_record": "Command A (FC)" }, { "id": "98c36396d1222566805e", "model": "BitAgent-Bounty-8B", "organization": "Bittensor", "configuration": "", "score": 0.4623, "raw_score": 46.23, "raw_scale": 100, "source_record": "BitAgent-Bounty-8B" }, { "id": "f11520d87f823729829e", "model": "Arch-Agent-32B", "organization": "katanemo", "configuration": "", "score": 0.4537, "raw_score": 45.37, "raw_scale": 100, "source_record": "Arch-Agent-32B" }, { "id": "2d8c83a6c0e6056a8698", "model": "GPT-5.2-2025-12-11 (Prompt)", "organization": "OpenAI", "configuration": "", "score": 0.45270000000000005, "raw_score": 45.27, "raw_scale": 100, "source_record": "GPT-5.2-2025-12-11 (Prompt)" }, { "id": "c95ade13ed35283b1a6c", "model": "Qwen3-8B (FC)", "organization": "Qwen", "configuration": "", "score": 0.4257, "raw_score": 42.57, "raw_scale": 100, "source_record": "Qwen3-8B (FC)" }, { "id": "1be2cd5e5b66d6de90e5", "model": "ToolACE-2-8B (FC)", "organization": "Huawei Noah & USTC", "configuration": "", "score": 0.4244, "raw_score": 42.44, "raw_scale": 100, "source_record": "ToolACE-2-8B (FC)" }, { "id": "c8c2237eeaac1d92abd0", "model": "Qwen3-30B-A3B-Instruct-2507 (FC)", "organization": "Qwen", "configuration": "", "score": 0.4139, "raw_score": 41.39, "raw_scale": 100, "source_record": "Qwen3-30B-A3B-Instruct-2507 (FC)" }, { "id": "f146706c6f53e16a1343", "model": "xLAM-2-3b-fc-r (FC)", "organization": "Salesforce", "configuration": "", "score": 0.4122, "raw_score": 41.22, "raw_scale": 100, "source_record": "xLAM-2-3b-fc-r (FC)" }, { "id": "341487607004b2c24373", "model": "Qwen3-14B (FC)", "organization": "Qwen", "configuration": "", "score": 0.4103, "raw_score": 41.03, "raw_scale": 100, "source_record": "Qwen3-14B (FC)" }, { "id": "54e94cd950347b135e4e", "model": "Qwen3-8B (Prompt)", "organization": "Qwen", "configuration": "", "score": 0.4043, "raw_score": 40.43, "raw_scale": 100, "source_record": "Qwen3-8B (Prompt)" }, { "id": "5c65eb6ab8a6ad26a2df", "model": "GPT-4.1-2025-04-14 (Prompt)", "organization": "OpenAI", "configuration": "", "score": 0.39380000000000004, "raw_score": 39.38, "raw_scale": 100, "source_record": "GPT-4.1-2025-04-14 (Prompt)" }, { "id": "d9b758588caedb377f61", "model": "mistral-large-2411 (FC)", "organization": "Mistral AI", "configuration": "", "score": 0.3837, "raw_score": 38.37, "raw_scale": 100, "source_record": "mistral-large-2411 (FC)" }, { "id": "218cdf20d014c00505e1", "model": "Qwen3-14B (Prompt)", "organization": "Qwen", "configuration": "", "score": 0.37770000000000004, "raw_score": 37.77, "raw_scale": 100, "source_record": "Qwen3-14B (Prompt)" }, { "id": "01bfd0aaf7702771667d", "model": "Mistral-Medium-2505", "organization": "Mistral AI", "configuration": "", "score": 0.37689999999999996, "raw_score": 37.69, "raw_scale": 100, "source_record": "Mistral-Medium-2505" }, { "id": "43a496fca5fd65fe8215", "model": "Mistral-Medium-2505 (FC)", "organization": "Mistral AI", "configuration": "", "score": 0.37560000000000004, "raw_score": 37.56, "raw_scale": 100, "source_record": "Mistral-Medium-2505 (FC)" }, { "id": "32c220de6de92220c324", "model": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", "organization": "Meta", "configuration": "", "score": 0.3729, "raw_score": 37.29, "raw_scale": 100, "source_record": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)" }, { "id": "df667ca39ece46a03265", "model": "Mistral-small-2506 (FC)", "organization": "Mistral AI", "configuration": "", "score": 0.3715, "raw_score": 37.15, "raw_scale": 100, "source_record": "Mistral-small-2506 (FC)" }, { "id": "a73b655d2c8fc395759e", "model": "Gemini-2.5-Flash-Lite (FC)", "organization": "Google", "configuration": "", "score": 0.36869999999999997, "raw_score": 36.87, "raw_scale": 100, "source_record": "Gemini-2.5-Flash-Lite (FC)" }, { "id": "3038933d969bf655af2e", "model": "Qwen3-30B-A3B-Instruct-2507 (Prompt)", "organization": "Qwen", "configuration": "", "score": 0.36700000000000005, "raw_score": 36.7, "raw_scale": 100, "source_record": "Qwen3-30B-A3B-Instruct-2507 (Prompt)" }, { "id": "4cffe47a2472cae6ce7f", "model": "Qwen3-4B-Instruct-2507 (FC)", "organization": "Qwen", "configuration": "", "score": 0.3568, "raw_score": 35.68, "raw_scale": 100, "source_record": "Qwen3-4B-Instruct-2507 (FC)" }, { "id": "7f28bd06a935aad78529", "model": "Qwen3-4B-Instruct-2507 (Prompt)", "organization": "Qwen", "configuration": "", "score": 0.3552, "raw_score": 35.52, "raw_scale": 100, "source_record": "Qwen3-4B-Instruct-2507 (Prompt)" }, { "id": "cc1c0fef9ffb087971ae", "model": "Arch-Agent-3B", "organization": "katanemo", "configuration": "", "score": 0.35359999999999997, "raw_score": 35.36, "raw_scale": 100, "source_record": "Arch-Agent-3B" }, { "id": "0c86b12d2dac8dc52fab", "model": "Claude-Opus-4-5-20251101 (Prompt)", "organization": "Anthropic", "configuration": "", "score": 0.3347, "raw_score": 33.47, "raw_scale": 100, "source_record": "Claude-Opus-4-5-20251101 (Prompt)" }, { "id": "218e880f48a924b64627", "model": "GPT-4.1-nano-2025-04-14 (FC)", "organization": "OpenAI", "configuration": "", "score": 0.33049999999999996, "raw_score": 33.05, "raw_scale": 100, "source_record": "GPT-4.1-nano-2025-04-14 (FC)" }, { "id": "e634d85fe1668513a3fb", "model": "Mistral-Small-2506 (Prompt)", "organization": "Mistral AI", "configuration": "", "score": 0.32380000000000003, "raw_score": 32.38, "raw_scale": 100, "source_record": "Mistral-Small-2506 (Prompt)" }, { "id": "771ccadf6bde779bc5a2", "model": "Arch-Agent-1.5B", "organization": "katanemo", "configuration": "", "score": 0.3214, "raw_score": 32.14, "raw_scale": 100, "source_record": "Arch-Agent-1.5B" }, { "id": "231db289d607496ccf10", "model": "Command R7B (FC)", "organization": "Cohere", "configuration": "", "score": 0.3207, "raw_score": 32.07, "raw_scale": 100, "source_record": "Command R7B (FC)" }, { "id": "54e1a618a9ab85c7663c", "model": "Llama-3.3-70B-Instruct (FC)", "organization": "Meta", "configuration": "", "score": 0.319, "raw_score": 31.9, "raw_scale": 100, "source_record": "Llama-3.3-70B-Instruct (FC)" }, { "id": "7f521bfad6958da9dcbb", "model": "mistral-large-2411 (Prompt)", "organization": "Mistral AI", "configuration": "", "score": 0.3184, "raw_score": 31.84, "raw_scale": 100, "source_record": "mistral-large-2411 (Prompt)" }, { "id": "033974bef014c4d4cc0b", "model": "Hammer2.1-7b (FC)", "organization": "MadeAgents", "configuration": "", "score": 0.31670000000000004, "raw_score": 31.67, "raw_scale": 100, "source_record": "Hammer2.1-7b (FC)" }, { "id": "6a9dafad604787e9da72", "model": "xLAM-2-1b-fc-r (FC)", "organization": "Salesforce", "configuration": "", "score": 0.3044, "raw_score": 30.44, "raw_scale": 100, "source_record": "xLAM-2-1b-fc-r (FC)" }, { "id": "a4475b509096638dfd91", "model": "Gemma-3-12b-it (Prompt)", "organization": "Google", "configuration": "", "score": 0.3043, "raw_score": 30.43, "raw_scale": 100, "source_record": "Gemma-3-12b-it (Prompt)" }, { "id": "85adcc3e30fb5cb67407", "model": "GPT-4.1-mini-2025-04-14 (Prompt)", "organization": "OpenAI", "configuration": "", "score": 0.2973, "raw_score": 29.73, "raw_scale": 100, "source_record": "GPT-4.1-mini-2025-04-14 (Prompt)" }, { "id": "5ccf7f5d7b20bcf233ac", "model": "Hammer2.1-3b (FC)", "organization": "MadeAgents", "configuration": "", "score": 0.29710000000000003, "raw_score": 29.71, "raw_scale": 100, "source_record": "Hammer2.1-3b (FC)" }, { "id": "baaf7c2b1ab55e9b9f9d", "model": "Gemma-3-27b-it (Prompt)", "organization": "Google", "configuration": "", "score": 0.29469999999999996, "raw_score": 29.47, "raw_scale": 100, "source_record": "Gemma-3-27b-it (Prompt)" }, { "id": "05cbac496bb2a44f01d4", "model": "Phi-4 (Prompt)", "organization": "Microsoft", "configuration": "", "score": 0.2879, "raw_score": 28.79, "raw_scale": 100, "source_record": "Phi-4 (Prompt)" }, { "id": "523fb632e5f9a98e9586", "model": "Qwen3-1.7B (FC)", "organization": "Qwen", "configuration": "", "score": 0.2841, "raw_score": 28.41, "raw_scale": 100, "source_record": "Qwen3-1.7B (FC)" }, { "id": "233d78a18cc3085512e5", "model": "Llama-4-Scout-17B-16E-Instruct (FC)", "organization": "Meta", "configuration": "", "score": 0.2813, "raw_score": 28.13, "raw_scale": 100, "source_record": "Llama-4-Scout-17B-16E-Instruct (FC)" }, { "id": "33b13da3350f58e4a0a8", "model": "Gemini-2.5-Flash-Lite (Prompt)", "organization": "Google", "configuration": "", "score": 0.2803, "raw_score": 28.03, "raw_scale": 100, "source_record": "Gemini-2.5-Flash-Lite (Prompt)" }, { "id": "34bc09247db960c0291e", "model": "CoALM-70B", "organization": "UIUC + Oumi", "configuration": "", "score": 0.2799, "raw_score": 27.99, "raw_scale": 100, "source_record": "CoALM-70B" }, { "id": "e4aad2e6288e31b79cbd", "model": "Hammer2.1-1.5b (FC)", "organization": "MadeAgents", "configuration": "", "score": 0.2788, "raw_score": 27.88, "raw_scale": 100, "source_record": "Hammer2.1-1.5b (FC)" }, { "id": "af2a0bf50ae6bd37479b", "model": "palmyra-x-004 (FC)", "organization": "Writer", "configuration": "", "score": 0.2787, "raw_score": 27.87, "raw_scale": 100, "source_record": "palmyra-x-004 (FC)" }, { "id": "f4ce2546b68601c40613", "model": "GPT-5-mini-2025-08-07 (Prompt)", "organization": "OpenAI", "configuration": "", "score": 0.2783, "raw_score": 27.83, "raw_scale": 100, "source_record": "GPT-5-mini-2025-08-07 (Prompt)" }, { "id": "69db59476c1064649577", "model": "Open-Mistral-Nemo-2407 (FC)", "organization": "Mistral AI", "configuration": "", "score": 0.2763, "raw_score": 27.63, "raw_scale": 100, "source_record": "Open-Mistral-Nemo-2407 (FC)" }, { "id": "695d6c4f1d0b7dccbd26", "model": "GPT-5-nano-2025-08-07 (Prompt)", "organization": "OpenAI", "configuration": "", "score": 0.2755, "raw_score": 27.55, "raw_scale": 100, "source_record": "GPT-5-nano-2025-08-07 (Prompt)" }, { "id": "2a76b7c7d63a95190399", "model": "Amazon-Nova-2-Lite-v1:0 (FC)", "organization": "Amazon", "configuration": "", "score": 0.271, "raw_score": 27.1, "raw_scale": 100, "source_record": "Amazon-Nova-2-Lite-v1:0 (FC)" }, { "id": "17a04ec1cb69dcfdf72c", "model": "Granite-3.1-8B-Instruct (FC)", "organization": "IBM", "configuration": "", "score": 0.271, "raw_score": 27.1, "raw_scale": 100, "source_record": "Granite-3.1-8B-Instruct (FC)" }, { "id": "6beeaa7a7c1c633d24f0", "model": "Falcon3-10B-Instruct (FC)", "organization": "TII UAE", "configuration": "", "score": 0.2701, "raw_score": 27.01, "raw_scale": 100, "source_record": "Falcon3-10B-Instruct (FC)" }, { "id": "2a014adf8b127a6fd91e", "model": "Granite-3.2-8B-Instruct (FC)", "organization": "IBM", "configuration": "", "score": 0.2687, "raw_score": 26.87, "raw_scale": 100, "source_record": "Granite-3.2-8B-Instruct (FC)" }, { "id": "513bc74aa4d6a8134448", "model": "CoALM-8B", "organization": "UIUC + Oumi", "configuration": "", "score": 0.2681, "raw_score": 26.81, "raw_scale": 100, "source_record": "CoALM-8B" }, { "id": "e009ae34da3fe388bc1c", "model": "Llama-3.1-8B-Instruct (Prompt)", "organization": "Meta", "configuration": "", "score": 0.2583, "raw_score": 25.83, "raw_scale": 100, "source_record": "Llama-3.1-8B-Instruct (Prompt)" }, { "id": "f70ef3531a4517124626", "model": "MiniCPM3-4B-FC (FC)", "organization": "openbmb", "configuration": "", "score": 0.2555, "raw_score": 25.55, "raw_scale": 100, "source_record": "MiniCPM3-4B-FC (FC)" }, { "id": "f89a471961e0a078f911", "model": "Claude-Haiku-4-5-20251001 (Prompt)", "organization": "Anthropic", "configuration": "", "score": 0.2526, "raw_score": 25.26, "raw_scale": 100, "source_record": "Claude-Haiku-4-5-20251001 (Prompt)" }, { "id": "fc5622cb24ef8b187f79", "model": "Amazon-Nova-Pro-v1:0 (FC)", "organization": "Amazon", "configuration": "", "score": 0.24969999999999998, "raw_score": 24.97, "raw_scale": 100, "source_record": "Amazon-Nova-Pro-v1:0 (FC)" }, { "id": "588709ac2870585cd973", "model": "Claude-Sonnet-4-5-20250929 (Prompt)", "organization": "Anthropic", "configuration": "", "score": 0.249, "raw_score": 24.9, "raw_scale": 100, "source_record": "Claude-Sonnet-4-5-20250929 (Prompt)" }, { "id": "eb145be2125eac007b3e", "model": "GPT-4.1-nano-2025-04-14 (Prompt)", "organization": "OpenAI", "configuration": "", "score": 0.2488, "raw_score": 24.88, "raw_scale": 100, "source_record": "GPT-4.1-nano-2025-04-14 (Prompt)" }, { "id": "268c2762312765326f29", "model": "Falcon3-7B-Instruct (FC)", "organization": "TII UAE", "configuration": "", "score": 0.2403, "raw_score": 24.03, "raw_scale": 100, "source_record": "Falcon3-7B-Instruct (FC)" }, { "id": "63ce74d2b4035a4abc8a", "model": "Qwen3-0.6B (FC)", "organization": "Qwen", "configuration": "", "score": 0.23929999999999998, "raw_score": 23.93, "raw_scale": 100, "source_record": "Qwen3-0.6B (FC)" }, { "id": "54ec8f57cc1800e99f2e", "model": "Granite-20b-FunctionCalling (FC)", "organization": "IBM", "configuration": "", "score": 0.2323, "raw_score": 23.23, "raw_scale": 100, "source_record": "Granite-20b-FunctionCalling (FC)" }, { "id": "8ab40a34f35e71ca5c2e", "model": "Qwen3-0.6B (Prompt)", "organization": "Qwen", "configuration": "", "score": 0.2238, "raw_score": 22.38, "raw_scale": 100, "source_record": "Qwen3-0.6B (Prompt)" }, { "id": "b98400217682e1bfcfdf", "model": "Amazon-Nova-Micro-v1:0 (FC)", "organization": "Amazon", "configuration": "", "score": 0.2229, "raw_score": 22.29, "raw_scale": 100, "source_record": "Amazon-Nova-Micro-v1:0 (FC)" }, { "id": "47d4c1c64752af1283ae", "model": "RZN-T (Prompt)", "organization": "Phronetic AI", "configuration": "", "score": 0.2225, "raw_score": 22.25, "raw_scale": 100, "source_record": "RZN-T (Prompt)" }, { "id": "c3f443952121b75ade58", "model": "MiniCPM3-4B (Prompt)", "organization": "openbmb", "configuration": "", "score": 0.2208, "raw_score": 22.08, "raw_scale": 100, "source_record": "MiniCPM3-4B (Prompt)" }, { "id": "f3ac225aa85d1784d838", "model": "Llama-3.2-3B-Instruct (FC)", "organization": "Meta", "configuration": "", "score": 0.2195, "raw_score": 21.95, "raw_scale": 100, "source_record": "Llama-3.2-3B-Instruct (FC)" }, { "id": "e263fe475bb3d8355bae", "model": "Bielik-11B-v2.3-Instruct (Prompt)", "organization": "SpeakLeash & ACK Cyfronet AGH", "configuration": "", "score": 0.21899999999999997, "raw_score": 21.9, "raw_scale": 100, "source_record": "Bielik-11B-v2.3-Instruct (Prompt)" }, { "id": "2754c9f196694917cad5", "model": "Hammer2.1-0.5b (FC)", "organization": "MadeAgents", "configuration": "", "score": 0.2122, "raw_score": 21.22, "raw_scale": 100, "source_record": "Hammer2.1-0.5b (FC)" }, { "id": "ca9f2818b273bdfe0d03", "model": "Gemma-3-4b-it (Prompt)", "organization": "Google", "configuration": "", "score": 0.1962, "raw_score": 19.62, "raw_scale": 100, "source_record": "Gemma-3-4b-it (Prompt)" }, { "id": "cdb81b80545258de1e57", "model": "Open-Mistral-Nemo-2407 (Prompt)", "organization": "Mistral AI", "configuration": "", "score": 0.1931, "raw_score": 19.31, "raw_scale": 100, "source_record": "Open-Mistral-Nemo-2407 (Prompt)" }, { "id": "393d762f834464bca174", "model": "Granite-4.0-350m (FC)", "organization": "IBM", "configuration": "", "score": 0.1898, "raw_score": 18.98, "raw_scale": 100, "source_record": "Granite-4.0-350m (FC)" }, { "id": "1f528611f3edfbe7fc73", "model": "Falcon3-3B-Instruct (FC)", "organization": "TII UAE", "configuration": "", "score": 0.1625, "raw_score": 16.25, "raw_scale": 100, "source_record": "Falcon3-3B-Instruct (FC)" }, { "id": "99ddd031bbf78a7e51b0", "model": "Ministral-8B-Instruct-2410 (FC)", "organization": "Mistral AI", "configuration": "", "score": 0.111, "raw_score": 11.1, "raw_scale": 100, "source_record": "Ministral-8B-Instruct-2410 (FC)" }, { "id": "ac5e00332f91913fe063", "model": "Falcon3-1B-Instruct (FC)", "organization": "TII UAE", "configuration": "", "score": 0.1108, "raw_score": 11.08, "raw_scale": 100, "source_record": "Falcon3-1B-Instruct (FC)" }, { "id": "292821a4d23d8722cc14", "model": "Llama-3.2-1B-Instruct (FC)", "organization": "Meta", "configuration": "", "score": 0.1082, "raw_score": 10.82, "raw_scale": 100, "source_record": "Llama-3.2-1B-Instruct (FC)" }, { "id": "162169a2b142039bd1db", "model": "Llama-3.1-Nemotron-Ultra-253B-v1 (FC)", "organization": "NVIDIA", "configuration": "", "score": 0.1, "raw_score": 10.0, "raw_scale": 100, "source_record": "Llama-3.1-Nemotron-Ultra-253B-v1 (FC)" }, { "id": "ebcb15953f23b9af4d02", "model": "Gemma-3-1b-it (Prompt)", "organization": "Google", "configuration": "", "score": 0.0717, "raw_score": 7.17, "raw_scale": 100, "source_record": "Gemma-3-1b-it (Prompt)" } ] }, { "id": "osworld-2-v2-1-full", "name": "OSWorld 2.0 (v2.1, full)", "group": "Agent 与工具", "unit": "%", "metric": "Binary task completion", "protocol": { "en": "Official end-to-end model runs, 500-step budget. Score is full task completion, not partial reward; reasoning settings are retained.", "zh": "官方端到端模型运行，最多 500 步。使用完整任务完成率，不使用部分奖励；保留推理设置。" }, "source": { "name": "OSWorld / XLANG Lab", "url": "https://osworld-v2.xlang.ai/", "data_url": "https://osworld-v2.xlang.ai/static/data/leaderboard/official-results.json?v=leaderboard-musespark-v10", "retrieved_at": "2026-11-01T16:45:15.576914+00:00", "sha256": "4cdcebeb33cd9c61cecfb275debc292326b48c209483d7da3ad66ee33d94b974", "selector": "v2.1; full; 500 steps; official=true; modelType=e2e; binaryAccuracy / 100", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "55ee50f66ec708b1882b", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "max; batch tool; 500 steps", "score": 0.44329999999999997, "raw_score": 44.33, "raw_scale": 100, "source_record": "v2.1|full|Claude Opus 5|max; batch tool; 500 steps" }, { "id": "19b3b57f698123bcc9f4", "model": "MuseSpark 1.3", "organization": "Meta", "configuration": "max; batch tool; 500 steps", "score": 0.3925, "raw_score": 39.25, "raw_scale": 100, "source_record": "v2.1|full|MuseSpark 1.3|max; batch tool; 500 steps" }, { "id": "62a07219d3407de4aa20", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "high; batch tool; 500 steps", "score": 0.3689, "raw_score": 36.89, "raw_scale": 100, "source_record": "v2.1|full|Claude Opus 5|high; batch tool; 500 steps" }, { "id": "e61604c46c92699c5666", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "xhigh; batch tool; 500 steps", "score": 0.3333, "raw_score": 33.33, "raw_scale": 100, "source_record": "v2.1|full|Claude Opus 5|xhigh; batch tool; 500 steps" }, { "id": "03143adeff6cef4f0d74", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "medium; batch tool; 500 steps", "score": 0.3301, "raw_score": 33.01, "raw_scale": 100, "source_record": "v2.1|full|Claude Opus 5|medium; batch tool; 500 steps" }, { "id": "f8cf2e79253cd531f2c7", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "low; batch tool; 500 steps", "score": 0.1881, "raw_score": 18.81, "raw_scale": 100, "source_record": "v2.1|full|Claude Opus 5|low; batch tool; 500 steps" } ] }, { "id": "osworld-2-20260808-offline", "name": "OSWorld 2.0 (2026.08.08, offline)", "group": "Agent 与工具", "unit": "%", "metric": "Binary task completion", "protocol": { "en": "Official end-to-end model runs, 500-step budget. Score is full task completion, not partial reward; reasoning settings are retained.", "zh": "官方端到端模型运行，最多 500 步。使用完整任务完成率，不使用部分奖励；保留推理设置。" }, "source": { "name": "OSWorld / XLANG Lab", "url": "https://osworld-v2.xlang.ai/", "data_url": "https://osworld-v2.xlang.ai/static/data/leaderboard/official-results.json?v=leaderboard-musespark-v10", "retrieved_at": "2026-11-01T16:45:15.576914+00:00", "sha256": "4cdcebeb33cd9c61cecfb275debc292326b48c209483d7da3ad66ee33d94b974", "selector": "v2026.08.08; offline; 500 steps; official=true; modelType=e2e; binaryAccuracy / 100", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "50d44937a72c88b383ae", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "max; batch tool; 500 steps", "score": 0.3472, "raw_score": 34.72, "raw_scale": 100, "source_record": "v2026.08.08|offline|Claude Opus 5|max; batch tool; 500 steps" }, { "id": "d10f737cf70a5217b34e", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "xhigh; batch tool; 500 steps", "score": 0.33380000000000004, "raw_score": 33.38, "raw_scale": 100, "source_record": "v2026.08.08|offline|Claude Opus 5|xhigh; batch tool; 500 steps" }, { "id": "94a61f4ddbf738949e67", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "high; batch tool; 500 steps", "score": 0.32689999999999997, "raw_score": 32.69, "raw_scale": 100, "source_record": "v2026.08.08|offline|Claude Opus 5|high; batch tool; 500 steps" }, { "id": "9f8f3d43d8dc06504030", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "max; batch tool; 500 steps", "score": 0.281, "raw_score": 28.1, "raw_scale": 100, "source_record": "v2026.08.08|offline|GPT-5.6 Sol|max; batch tool; 500 steps" }, { "id": "09630d66cf27b51e81d3", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "medium; batch tool; 500 steps", "score": 0.2713, "raw_score": 27.13, "raw_scale": 100, "source_record": "v2026.08.08|offline|Claude Opus 5|medium; batch tool; 500 steps" }, { "id": "5d37396bcc83f5b6cbf6", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "xhigh; batch tool; 500 steps", "score": 0.2463, "raw_score": 24.63, "raw_scale": 100, "source_record": "v2026.08.08|offline|GPT-5.6 Sol|xhigh; batch tool; 500 steps" }, { "id": "5e4f37d610c2d381c452", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "low; batch tool; 500 steps", "score": 0.24609999999999999, "raw_score": 24.61, "raw_scale": 100, "source_record": "v2026.08.08|offline|Claude Opus 5|low; batch tool; 500 steps" }, { "id": "ecc98056941b585b9001", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "high; batch tool; 500 steps", "score": 0.2, "raw_score": 20.0, "raw_scale": 100, "source_record": "v2026.08.08|offline|GPT-5.6 Sol|high; batch tool; 500 steps" }, { "id": "8429554541c2d81f1ad9", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "medium; batch tool; 500 steps", "score": 0.17559999999999998, "raw_score": 17.56, "raw_scale": 100, "source_record": "v2026.08.08|offline|GPT-5.6 Sol|medium; batch tool; 500 steps" }, { "id": "5f425e345d7d03536949", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "low; batch tool; 500 steps", "score": 0.0732, "raw_score": 7.32, "raw_scale": 100, "source_record": "v2026.08.08|offline|GPT-5.6 Sol|low; batch tool; 500 steps" }, { "id": "833ff8c66ed0dbd893be", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "none; batch tool; 500 steps", "score": 0.0341, "raw_score": 3.41, "raw_scale": 100, "source_record": "v2026.08.08|offline|GPT-5.6 Sol|none; batch tool; 500 steps" } ] }, { "id": "longbench-v2-cot", "name": "LongBench v2 (CoT)", "group": "指令与长文", "unit": "%", "metric": "Overall accuracy with CoT", "protocol": { "en": "Overall accuracy with chain-of-thought reasoning; missing CoT scores are omitted. Context limits follow each published entry.", "zh": "使用思维链设置的总体准确率，缺少该设置成绩的模型不计入；上下文上限沿用各条官方记录。" }, "source": { "name": "LongBench / THUDM", "url": "https://longbench2.github.io/", "data_url": "https://longbench2.github.io/", "retrieved_at": "2026-11-01T16:45:16.251042+00:00", "sha256": "b16f4683a0970694a1aa4fc74155bc61024fa57f98f257b90cc67d54d48d2d35", "selector": "#results: Overall w/ CoT column / 100; omit missing scores and Random baseline", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "d79f6610a7d805a69c4c", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "CoT; context 1M", "score": 0.633, "raw_score": 63.3, "raw_scale": 100, "source_record": "Gemini-2.5-Pro|2025-03-25|CoT", "source_date": "2025-03-25" }, { "id": "419bb8fa5ec8d4cdf03b", "model": "Gemini-2.5-Flash", "organization": "Google", "configuration": "CoT; context 1M", "score": 0.621, "raw_score": 62.1, "raw_scale": 100, "source_record": "Gemini-2.5-Flash|2025-04-17|CoT", "source_date": "2025-04-17" }, { "id": "14e8ee6b889c6195a893", "model": "Qwen3-235B-A22B-Thinking-2507", "organization": "Alibaba", "configuration": "CoT; context 256k", "score": 0.606, "raw_score": 60.6, "raw_scale": 100, "source_record": "Qwen3-235B-A22B-Thinking-2507|2025-07-25|CoT", "source_date": "2025-07-25" }, { "id": "4c43f58fb228af163233", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "CoT; context 128k", "score": 0.583, "raw_score": 58.3, "raw_scale": 100, "source_record": "DeepSeek-R1|2025-01-20|CoT", "source_date": "2025-01-20" }, { "id": "d865e1adb2880e16a5bd", "model": "Qwen3-235B-A22B-Instruct-2507", "organization": "Alibaba", "configuration": "CoT; context 256k", "score": 0.583, "raw_score": 58.3, "raw_scale": 100, "source_record": "Qwen3-235B-A22B-Instruct-2507|2025-07-22|CoT", "source_date": "2025-07-22" }, { "id": "1b6ce7ebcd790c853bc6", "model": "o1-preview", "organization": "OpenAI", "configuration": "CoT; context 128k", "score": 0.5770000000000001, "raw_score": 57.7, "raw_scale": 100, "source_record": "o1-preview|2024-09-12|CoT", "source_date": "2024-09-12" }, { "id": "97b8ec49bda91b14ad25", "model": "DeepSeek-R1-0528", "organization": "DeepSeek", "configuration": "CoT; context 128k", "score": 0.5670000000000001, "raw_score": 56.7, "raw_scale": 100, "source_record": "DeepSeek-R1-0528|2025-05-28|CoT", "source_date": "2025-05-28" }, { "id": "9de513c967de1c812e7f", "model": "MiniMax-Text-01", "organization": "MiniMax", "configuration": "CoT; context 4M", "score": 0.565, "raw_score": 56.5, "raw_scale": 100, "source_record": "MiniMax-Text-01|2025-01-15|CoT", "source_date": "2025-01-15" }, { "id": "59f164ca6a2afff964a9", "model": "Gemini-2.0-Flash-Thinking", "organization": "Google", "configuration": "CoT; context 1M", "score": 0.56, "raw_score": 56.0, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Thinking|2025-01-21|CoT", "source_date": "2025-01-21" }, { "id": "196b8fdad0b58537393c", "model": "Gemini-Exp-1206", "organization": "Google", "configuration": "CoT; context 2M", "score": 0.525, "raw_score": 52.5, "raw_scale": 100, "source_record": "Gemini-Exp-1206|2024-12-06|CoT", "source_date": "2024-12-06" }, { "id": "9b817a4f19a6da915d2b", "model": "GPT-4o (2024-11-20)", "organization": "OpenAI", "configuration": "CoT; context 128k", "score": 0.514, "raw_score": 51.4, "raw_scale": 100, "source_record": "GPT-4o|2024-11-20|CoT", "source_date": "2024-11-20" }, { "id": "746db6f70164bea23d5f", "model": "GPT-4o (2024-08-06)", "organization": "OpenAI", "configuration": "CoT; context 128k", "score": 0.512, "raw_score": 51.2, "raw_scale": 100, "source_record": "GPT-4o|2024-08-06|CoT", "source_date": "2024-08-06" }, { "id": "e4d3e8e9c1d1453d0a05", "model": "Gemini-2.0-Flash", "organization": "Google", "configuration": "CoT; context 1M", "score": 0.511, "raw_score": 51.1, "raw_scale": 100, "source_record": "Gemini-2.0-Flash|2024-12-11|CoT", "source_date": "2024-12-11" }, { "id": "8e8f991097f03776f97c", "model": "GLM-4.5", "organization": "Z.ai & Tsinghua", "configuration": "CoT; context 128k", "score": 0.503, "raw_score": 50.3, "raw_scale": 100, "source_record": "GLM-4.5|2025-07-28|CoT", "source_date": "2025-07-28" }, { "id": "f5dcdc1ec61ebf47de85", "model": "Qwen3-235B-A22B", "organization": "Alibaba", "configuration": "CoT; context 128k", "score": 0.501, "raw_score": 50.1, "raw_scale": 100, "source_record": "Qwen3-235B-A22B|2025-04-29|CoT", "source_date": "2025-04-29" }, { "id": "4769b2f820fc1249e6e1", "model": "Qwen3-30B-A3B-Thinking-2507", "organization": "Alibaba", "configuration": "CoT; context 256k", "score": 0.501, "raw_score": 50.1, "raw_scale": 100, "source_record": "Qwen3-30B-A3B-Thinking-2507|2025-07-25|CoT", "source_date": "2025-07-25" }, { "id": "a22b39e913ffa6e98ce6", "model": "Qwen3-32B", "organization": "Alibaba", "configuration": "CoT; context 128k", "score": 0.49200000000000005, "raw_score": 49.2, "raw_scale": 100, "source_record": "Qwen3-32B|2025-04-29|CoT", "source_date": "2025-04-29" }, { "id": "f5725f8cf27a5416a558", "model": "QwQ-32B", "organization": "Alibaba", "configuration": "CoT; context 128k", "score": 0.489, "raw_score": 48.9, "raw_scale": 100, "source_record": "QwQ-32B|2025-03-06|CoT", "source_date": "2025-03-06" }, { "id": "fdeda99078bc27e286d1", "model": "GLM-4.5-Air", "organization": "Z.ai & Tsinghua", "configuration": "CoT; context 128k", "score": 0.486, "raw_score": 48.6, "raw_scale": 100, "source_record": "GLM-4.5-Air|2025-07-28|CoT", "source_date": "2025-07-28" }, { "id": "5bff4f906381601b78b5", "model": "Claude 3.5 Sonnet", "organization": "Anthropic", "configuration": "CoT; context 200k", "score": 0.467, "raw_score": 46.7, "raw_scale": 100, "source_record": "Claude 3.5 Sonnet|2024-10-22|CoT", "source_date": "2024-10-22" }, { "id": "67c1aa4fd437b7e41f8a", "model": "GLM-4-Plus", "organization": "Z.ai & Tsinghua", "configuration": "CoT; context 128k", "score": 0.461, "raw_score": 46.1, "raw_scale": 100, "source_record": "GLM-4-Plus|2024-10-11|CoT", "source_date": "2024-10-11" }, { "id": "667e057fc73562cb092f", "model": "Kimi-K2-Instruct", "organization": "Moonshot AI", "configuration": "CoT; context 128k", "score": 0.44299999999999995, "raw_score": 44.3, "raw_scale": 100, "source_record": "Kimi-K2-Instruct|2025-07-11|CoT", "source_date": "2025-07-11" }, { "id": "8ad1845d1c7210989f52", "model": "Qwen2.5-72B", "organization": "Alibaba", "configuration": "CoT; context 128k", "score": 0.435, "raw_score": 43.5, "raw_scale": 100, "source_record": "Qwen2.5-72B|2024-09-19|CoT", "source_date": "2024-09-19" }, { "id": "07f67fbeadd4832e5fbc", "model": "Qwen3-30B-A3B", "organization": "Alibaba", "configuration": "CoT; context 128k", "score": 0.425, "raw_score": 42.5, "raw_scale": 100, "source_record": "Qwen3-30B-A3B|2025-04-29|CoT", "source_date": "2025-04-29" }, { "id": "e79b49675298b83d2f7e", "model": "Mistral Large 24.11", "organization": "Mistral AI", "configuration": "CoT; context 128k", "score": 0.396, "raw_score": 39.6, "raw_scale": 100, "source_record": "Mistral Large 24.11|2024-11-24|CoT", "source_date": "2024-11-24" }, { "id": "df9fe9354dca5bd23540", "model": "o1-mini", "organization": "OpenAI", "configuration": "CoT; context 128k", "score": 0.389, "raw_score": 38.9, "raw_scale": 100, "source_record": "o1-mini|2024-09-12|CoT", "source_date": "2024-09-12" }, { "id": "4594987f013b7d165425", "model": "Llama 3.1 70B", "organization": "Meta", "configuration": "CoT; context 128k", "score": 0.36200000000000004, "raw_score": 36.2, "raw_scale": 100, "source_record": "Llama 3.1 70B|2024-07-23|CoT", "source_date": "2024-07-23" }, { "id": "879e31fe0110a119b29f", "model": "Llama 3.3 70B", "organization": "Meta", "configuration": "CoT; context 128k", "score": 0.36200000000000004, "raw_score": 36.2, "raw_scale": 100, "source_record": "Llama 3.3 70B|2024-12-06|CoT", "source_date": "2024-12-06" }, { "id": "227810c601cb064ab05f", "model": "Qwen2.5-7B", "organization": "Alibaba", "configuration": "CoT; context 128k", "score": 0.35600000000000004, "raw_score": 35.6, "raw_scale": 100, "source_record": "Qwen2.5-7B|2024-09-19|CoT", "source_date": "2024-09-19" }, { "id": "1a8be2bb19ed7d684a76", "model": "Nemotron 70B", "organization": "Nvidia", "configuration": "CoT; context 128k", "score": 0.35200000000000004, "raw_score": 35.2, "raw_scale": 100, "source_record": "Nemotron 70B|2024-10-15|CoT", "source_date": "2024-10-15" }, { "id": "689995f6cf0505465fe5", "model": "Mistral Large 2", "organization": "Mistral AI", "configuration": "CoT; context 128k", "score": 0.336, "raw_score": 33.6, "raw_scale": 100, "source_record": "Mistral Large 2|2024-07-24|CoT", "source_date": "2024-07-24" }, { "id": "626f862ddddc57f939aa", "model": "GPT-4o mini", "organization": "OpenAI", "configuration": "CoT; context 128k", "score": 0.324, "raw_score": 32.4, "raw_scale": 100, "source_record": "GPT-4o mini|2024-07-18|CoT", "source_date": "2024-07-18" }, { "id": "b32ead473c8a02cce21f", "model": "NExtLong 8B", "organization": "CAS", "configuration": "CoT; context 512k", "score": 0.32, "raw_score": 32.0, "raw_scale": 100, "source_record": "NExtLong 8B|2025-01-23|CoT", "source_date": "2025-01-23" }, { "id": "d01e1943b98170793367", "model": "Command R+", "organization": "Cohere", "configuration": "CoT; context 128k", "score": 0.316, "raw_score": 31.6, "raw_scale": 100, "source_record": "Command R+|2024-08-30|CoT", "source_date": "2024-08-30" }, { "id": "3f3a7efcedcc122f00e6", "model": "GLM-4-9B", "organization": "Z.ai & Tsinghua", "configuration": "CoT; context 128k", "score": 0.308, "raw_score": 30.8, "raw_scale": 100, "source_record": "GLM-4-9B|2024-06-05|CoT", "source_date": "2024-06-05" }, { "id": "0e452d38dcb655422196", "model": "Llama 3.1 8B", "organization": "Meta", "configuration": "CoT; context 128k", "score": 0.304, "raw_score": 30.4, "raw_scale": 100, "source_record": "Llama 3.1 8B|2024-07-23|CoT", "source_date": "2024-07-23" } ] }, { "id": "video-mme-no-subs", "name": "Video-MME (no subtitles)", "group": "视觉与视频", "unit": "%", "metric": "Overall accuracy without subtitles", "protocol": { "en": "Overall accuracy without subtitles. Frame sampling settings are preserved for each model.", "zh": "不使用字幕的总体准确率；保留各模型的采样帧设置。" }, "source": { "name": "Video-MME", "url": "https://video-mme.github.io/home_page.html#leaderboard", "data_url": "https://raw.githubusercontent.com/Video-MME/video-mme.github.io/main/home_page.html", "retrieved_at": "2026-11-01T16:45:16.820494+00:00", "sha256": "02b88dcf1268ac331b79a3db7c1bdb1dde1d9f2389f424340ef6082c98adbe6d", "selector": "Overall w/o subs / 100; frame setting preserved", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "89300e93b51c279c01d8", "model": "video-SALMONN 2+", "organization": "THU & ByteDance", "configuration": "No subtitles; frames: 10 fps", "score": 0.797, "raw_score": 79.7, "raw_scale": 100, "source_record": "video-SALMONN 2+|No subtitles; frames: 10 fps", "source_date": "2025-09-28" }, { "id": "6a59a88b27c04b51d704", "model": "Gemini 1.5 Pro", "organization": "Google", "configuration": "No subtitles; frames: 1/0.5 fps", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "Gemini 1.5 Pro|No subtitles; frames: 1/0.5 fps", "source_date": "2024-06-15" }, { "id": "5ddc2fbef2c18b35c51c", "model": "AdaReTaKe", "organization": "HIT & Huawei", "configuration": "No subtitles; frames: 1024", "score": 0.735, "raw_score": 73.5, "raw_scale": 100, "source_record": "AdaReTaKe|No subtitles; frames: 1024", "source_date": "2025-03-04" }, { "id": "63f00eeb021f7357aff3", "model": "Keye-VL-1.5", "organization": "Kwai Keye Team", "configuration": "No subtitles; frames: 64", "score": 0.73, "raw_score": 73.0, "raw_scale": 100, "source_record": "Keye-VL-1.5|No subtitles; frames: 64", "source_date": "2025-08-29" }, { "id": "6886bcebe3fb174f5721", "model": "TSPO", "organization": "XJTU & TeleAI & USTB", "configuration": "No subtitles; frames: 128", "score": 0.73, "raw_score": 73.0, "raw_scale": 100, "source_record": "TSPO|No subtitles; frames: 128", "source_date": "2025-09-22" }, { "id": "6ed30d819c3e3a315d89", "model": "InternVL2.5", "organization": "Shanghai AI Lab", "configuration": "No subtitles; frames: 64", "score": 0.721, "raw_score": 72.1, "raw_scale": 100, "source_record": "InternVL2.5|No subtitles; frames: 64", "source_date": "2024-12-13" }, { "id": "4d991420a6d8b32ef0bc", "model": "GPT-4o", "organization": "OpenAI", "configuration": "No subtitles; frames: 384", "score": 0.7190000000000001, "raw_score": 71.9, "raw_scale": 100, "source_record": "GPT-4o|No subtitles; frames: 384", "source_date": "2024-06-15" }, { "id": "331b31f8f910768ec4dc", "model": "Qwen2-VL", "organization": "Alibaba", "configuration": "No subtitles; frames: 768", "score": 0.7120000000000001, "raw_score": 71.2, "raw_scale": 100, "source_record": "Qwen2-VL|No subtitles; frames: 768", "source_date": "2024-08-19" }, { "id": "7fc43215008369d082eb", "model": "JT-VL-Chat", "organization": "China Mobile", "configuration": "No subtitles; frames: 1800", "score": 0.706, "raw_score": 70.6, "raw_scale": 100, "source_record": "JT-VL-Chat|No subtitles; frames: 1800", "source_date": "2025-07-24" }, { "id": "a7d35d3f01e893ad1931", "model": "LLaVA-Video", "organization": "Bytedance & NTU S-Lab", "configuration": "No subtitles; frames: 64", "score": 0.706, "raw_score": 70.6, "raw_scale": 100, "source_record": "LLaVA-Video|No subtitles; frames: 64", "source_date": "2024-08-28" }, { "id": "cbf55fa9b352763cdaf8", "model": "Gemini 1.5 Flash", "organization": "Google", "configuration": "No subtitles; frames: 1/0.5 fps", "score": 0.703, "raw_score": 70.3, "raw_scale": 100, "source_record": "Gemini 1.5 Flash|No subtitles; frames: 1/0.5 fps", "source_date": "2024-06-15" }, { "id": "8d656ab34f0b17679924", "model": "LinVT", "organization": "Meituan Inc", "configuration": "No subtitles; frames: 120", "score": 0.703, "raw_score": 70.3, "raw_scale": 100, "source_record": "LinVT|No subtitles; frames: 120", "source_date": "2025-02-05" }, { "id": "acfb9de1fba2a149916e", "model": "Aria", "organization": "Rhymes AI", "configuration": "No subtitles; frames: 256", "score": 0.6759999999999999, "raw_score": 67.6, "raw_scale": 100, "source_record": "Aria|No subtitles; frames: 256", "source_date": "2024-10-11" }, { "id": "b1a35c472be424e0ba1a", "model": "ViLAMP", "organization": "RUC & Ant Group", "configuration": "No subtitles; frames: 1 fps", "score": 0.675, "raw_score": 67.5, "raw_scale": 100, "source_record": "ViLAMP|No subtitles; frames: 1 fps", "source_date": "2025-04-02" }, { "id": "5f585e3bb64dd15e7a20", "model": "Oryx-1.5", "organization": "THU & Tencent & NTU", "configuration": "No subtitles; frames: 128", "score": 0.6729999999999999, "raw_score": 67.3, "raw_scale": 100, "source_record": "Oryx-1.5|No subtitles; frames: 128", "source_date": "2024-10-21" }, { "id": "8bd1e7e745634b25cfee", "model": "Long-VITA", "organization": "Tencent & NJU", "configuration": "No subtitles; frames: 1 fps", "score": 0.67, "raw_score": 67.0, "raw_scale": 100, "source_record": "Long-VITA|No subtitles; frames: 1 fps", "source_date": "2025-07-23" }, { "id": "0d7135063f59bf614bd3", "model": "LLaVA-OneVision", "organization": "Bytedance & NTU S-Lab", "configuration": "No subtitles; frames: 32", "score": 0.6629999999999999, "raw_score": 66.3, "raw_scale": 100, "source_record": "LLaVA-OneVision|No subtitles; frames: 32", "source_date": "2024-08-08" }, { "id": "f5c5ca0e24056a5cc557", "model": "VideoLLaMA 3", "organization": "Alibaba", "configuration": "No subtitles; frames: 180", "score": 0.662, "raw_score": 66.2, "raw_scale": 100, "source_record": "VideoLLaMA 3|No subtitles; frames: 180", "source_date": "2025-01-24" }, { "id": "9a56128f31febfe90d89", "model": "QuoTA", "organization": "XMU", "configuration": "No subtitles; frames: 64", "score": 0.659, "raw_score": 65.9, "raw_scale": 100, "source_record": "QuoTA|No subtitles; frames: 64", "source_date": "2025-03-04" }, { "id": "7034b52278b2941288bf", "model": "TPO", "organization": "Stanford University", "configuration": "No subtitles; frames: 96", "score": 0.6559999999999999, "raw_score": 65.6, "raw_scale": 100, "source_record": "TPO|No subtitles; frames: 96", "source_date": "2025-01-26" }, { "id": "f01effb157c438e80fbe", "model": "VideoChat-Flash", "organization": "Shanghai AI Lab & NJU", "configuration": "No subtitles; frames: 512", "score": 0.653, "raw_score": 65.3, "raw_scale": 100, "source_record": "VideoChat-Flash|No subtitles; frames: 512", "source_date": "2025-02-06" }, { "id": "451796a32ade9ef2b4ca", "model": "GPT-4o mini", "organization": "OpenAI", "configuration": "No subtitles; frames: 250", "score": 0.648, "raw_score": 64.8, "raw_scale": 100, "source_record": "GPT-4o mini|No subtitles; frames: 250", "source_date": "2024-07-21" }, { "id": "69569fac9ffdc81e0b3a", "model": "ByteVideoLLM", "organization": "Bytedance", "configuration": "No subtitles; frames: 100", "score": 0.6459999999999999, "raw_score": 64.6, "raw_scale": 100, "source_record": "ByteVideoLLM|No subtitles; frames: 100", "source_date": "2024-10-21" }, { "id": "f4a2d7b10af332eeb22e", "model": "NVILA", "organization": "NVIDIA", "configuration": "No subtitles; frames: 1024", "score": 0.642, "raw_score": 64.2, "raw_scale": 100, "source_record": "NVILA|No subtitles; frames: 1024", "source_date": "2024-11-06" }, { "id": "a6bee74ab2b36efca2f9", "model": "LiveCC", "organization": "NUS Show Lab & ByteDance", "configuration": "No subtitles; frames: 2 fps", "score": 0.6409999999999999, "raw_score": 64.1, "raw_scale": 100, "source_record": "LiveCC|No subtitles; frames: 2 fps", "source_date": "2025-04-26" }, { "id": "8edda041dcb20e5469ef", "model": "MiniCPM-o 2.6", "organization": "OpenBMB", "configuration": "No subtitles; frames: 64", "score": 0.639, "raw_score": 63.9, "raw_scale": 100, "source_record": "MiniCPM-o 2.6|No subtitles; frames: 64", "source_date": "2025-01-16" }, { "id": "b4063de7256713200e31", "model": "ReTaKe", "organization": "HIT & Huawei", "configuration": "No subtitles; frames: 1024", "score": 0.639, "raw_score": 63.9, "raw_scale": 100, "source_record": "ReTaKe|No subtitles; frames: 1024", "source_date": "2025-02-05" }, { "id": "7a916ef9024c218f8ae3", "model": "VideoLLaMA 2", "organization": "Alibaba", "configuration": "No subtitles; frames: 32", "score": 0.624, "raw_score": 62.4, "raw_scale": 100, "source_record": "VideoLLaMA 2|No subtitles; frames: 32", "source_date": "2024-08-29" }, { "id": "1d0b3e51ce256d045d62", "model": "InternVL2", "organization": "Shanghai AI Lab", "configuration": "No subtitles; frames: 16", "score": 0.612, "raw_score": 61.2, "raw_scale": 100, "source_record": "InternVL2|No subtitles; frames: 16", "source_date": "2024-07-18" }, { "id": "39f05747759acf57c451", "model": "MiniCPM-V 2.6", "organization": "OpenBMB", "configuration": "No subtitles; frames: 64", "score": 0.609, "raw_score": 60.9, "raw_scale": 100, "source_record": "MiniCPM-V 2.6|No subtitles; frames: 64", "source_date": "2024-08-12" }, { "id": "be11c953874ae07801a8", "model": "Claude 3.5 Sonnet", "organization": "Anthropic", "configuration": "No subtitles; frames: 20", "score": 0.6, "raw_score": 60.0, "raw_scale": 100, "source_record": "Claude 3.5 Sonnet|No subtitles; frames: 20", "source_date": "2024-07-30" }, { "id": "8d5eb110bd3529deaeb0", "model": "GPT-4V", "organization": "OpenAI", "configuration": "No subtitles; frames: 10", "score": 0.599, "raw_score": 59.9, "raw_scale": 100, "source_record": "GPT-4V|No subtitles; frames: 10", "source_date": "2024-06-15" }, { "id": "e365b95dc9e102ebf045", "model": "mPLUG-Owl3", "organization": "Alibaba", "configuration": "No subtitles; frames: 128", "score": 0.593, "raw_score": 59.3, "raw_scale": 100, "source_record": "mPLUG-Owl3|No subtitles; frames: 128", "source_date": "2024-11-13" }, { "id": "bf1a415a20c566d068b8", "model": "TimeMarker", "organization": "Meituan", "configuration": "No subtitles; frames: 128", "score": 0.573, "raw_score": 57.3, "raw_scale": 100, "source_record": "TimeMarker|No subtitles; frames: 128", "source_date": "2024-11-07" }, { "id": "284bf34b0d5b50e1710b", "model": "VITA 1.5", "organization": "Tencent Youtu Lab & NJU", "configuration": "No subtitles; frames: 16", "score": 0.561, "raw_score": 56.1, "raw_scale": 100, "source_record": "VITA 1.5|No subtitles; frames: 16", "source_date": "2025-01-22" }, { "id": "4873481c835a55c9bc74", "model": "Kangaroo", "organization": "Meituan & UCAS", "configuration": "No subtitles; frames: 64", "score": 0.56, "raw_score": 56.0, "raw_scale": 100, "source_record": "Kangaroo|No subtitles; frames: 64", "source_date": "2024-07-23" }, { "id": "c1126cf3ea6919b880f3", "model": "VITA", "organization": "Tencent Youtu Lab & NJU", "configuration": "No subtitles; frames: 32", "score": 0.5579999999999999, "raw_score": 55.8, "raw_scale": 100, "source_record": "VITA|No subtitles; frames: 32", "source_date": "2024-09-08" }, { "id": "4b65ff968e876a9d7847", "model": "Video-XL", "organization": "SJTU & BAAI", "configuration": "No subtitles; frames: 128", "score": 0.555, "raw_score": 55.5, "raw_scale": 100, "source_record": "Video-XL|No subtitles; frames: 128", "source_date": "2024-10-24" }, { "id": "e7efe320ea65c715d8e4", "model": "Video-CCAM", "organization": "QQMM", "configuration": "No subtitles; frames: 96", "score": 0.532, "raw_score": 53.2, "raw_scale": 100, "source_record": "Video-CCAM|No subtitles; frames: 96", "source_date": "2024-07-16" }, { "id": "75da6a0af61f0b200907", "model": "Long-LLaVA", "organization": "Amazon", "configuration": "No subtitles; frames: 64", "score": 0.529, "raw_score": 52.9, "raw_scale": 100, "source_record": "Long-LLaVA|No subtitles; frames: 64", "source_date": "2024-09-09" }, { "id": "81ae7b975ef56281da8c", "model": "LongVA", "organization": "NTU S-Lab", "configuration": "No subtitles; frames: 128", "score": 0.526, "raw_score": 52.6, "raw_scale": 100, "source_record": "LongVA|No subtitles; frames: 128", "source_date": "2024-06-25" }, { "id": "4236b42980c25801276c", "model": "Qwen-VL-Max", "organization": "Alibaba", "configuration": "No subtitles; frames: 4", "score": 0.513, "raw_score": 51.3, "raw_scale": 100, "source_record": "Qwen-VL-Max|No subtitles; frames: 4", "source_date": "2024-06-15" }, { "id": "1b4b6e236ca92478d732", "model": "InternVL-Chat-V1.5", "organization": "Shanghai AI Lab", "configuration": "No subtitles; frames: 10", "score": 0.507, "raw_score": 50.7, "raw_scale": 100, "source_record": "InternVL-Chat-V1.5|No subtitles; frames: 10", "source_date": "2024-06-15" }, { "id": "561b7fdfe261b775c2f7", "model": "SliME", "organization": "CASIA", "configuration": "No subtitles; frames: 8", "score": 0.45299999999999996, "raw_score": 45.3, "raw_scale": 100, "source_record": "SliME|No subtitles; frames: 8", "source_date": "2024-07-16" }, { "id": "2087b734958a2d90a6b9", "model": "ShareGemini", "organization": "XMU", "configuration": "No subtitles; frames: 64", "score": 0.43200000000000005, "raw_score": 43.2, "raw_scale": 100, "source_record": "ShareGemini|No subtitles; frames: 64", "source_date": "2024-06-20" }, { "id": "b43f1bce78ab3e638b32", "model": "Qwen-VL-Chat", "organization": "Alibaba", "configuration": "No subtitles; frames: 4", "score": 0.41100000000000003, "raw_score": 41.1, "raw_scale": 100, "source_record": "Qwen-VL-Chat|No subtitles; frames: 4", "source_date": "2024-06-15" }, { "id": "376eea0239c8eae79786", "model": "Chat-UniVi-v1.5", "organization": "PKU", "configuration": "No subtitles; frames: 64", "score": 0.406, "raw_score": 40.6, "raw_scale": 100, "source_record": "Chat-UniVi-v1.5|No subtitles; frames: 64", "source_date": "2024-06-15" }, { "id": "286d39a518114e739bb3", "model": "ShareGPT4Video", "organization": "Shanghai AI Lab", "configuration": "No subtitles; frames: 16", "score": 0.39899999999999997, "raw_score": 39.9, "raw_scale": 100, "source_record": "ShareGPT4Video|No subtitles; frames: 16", "source_date": "2024-06-17" }, { "id": "f4bb74e08a9b773279b7", "model": "Video-LLaVA", "organization": "PKU", "configuration": "No subtitles; frames: 8", "score": 0.39899999999999997, "raw_score": 39.9, "raw_scale": 100, "source_record": "Video-LLaVA|No subtitles; frames: 8", "source_date": "2024-06-15" }, { "id": "0f059cd9f879ec4e3e76", "model": "VideoChat2-Mistral", "organization": "Shanghai AI Lab", "configuration": "No subtitles; frames: 16", "score": 0.395, "raw_score": 39.5, "raw_scale": 100, "source_record": "VideoChat2-Mistral|No subtitles; frames: 16", "source_date": "2024-06-15" }, { "id": "4900acc87de055e2d96c", "model": "ST-LLM", "organization": "PKU", "configuration": "No subtitles; frames: 64", "score": 0.379, "raw_score": 37.9, "raw_scale": 100, "source_record": "ST-LLM|No subtitles; frames: 64", "source_date": "2024-06-15" } ] }, { "id": "gaia-2023-test", "name": "GAIA (2023 test)", "group": "Agent 与工具", "unit": "%", "metric": "Task success", "protocol": { "en": "301 test tasks. Named submissions from the official top 100 plus identifiable submissions with external project links in the full export. Teams report their agent and base models.", "zh": "301 道测试题。收录官方展示的前 100 条中身份完整的提交，以及全量文件中带外部项目链接的具名提交。Agent 和基础模型由提交方填写。" }, "source": { "data_url": "https://huggingface.co/datasets/gaia-benchmark/results_public/resolve/main/2023/test-00000-of-00001.parquet", "sha256": "a0e41a10442797748ce997d6de541b60bc84c8bae19b46e2237058a8a56dbaa0", "retrieved_at": "2026-11-02T05:57:37.045193+00:00", "name": "GAIA authors", "url": "https://huggingface.co/spaces/gaia-benchmark/leaderboard", "selector": "2023/test; named agent, submitter and base models; official top-100 display or linked external project; latest identical configuration", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "fd412a45fac317feb0ea", "model": "Keen AgenticOS V1.0", "organization": "KeenData", "configuration": "Base models: Deepseek，Kimi，Claude", "score": 0.9401993355481728, "raw_score": 0.9401993355481728, "raw_scale": 1, "source_record": "Keen AgenticOS V1.0|Deepseek，Kimi，Claude|2026-10-29", "source_date": "2026-10-29", "reported_by": "KeenData" }, { "id": "8f6033dce17379efb2f5", "model": "Shadow Harness V8.01", "organization": "Shadow", "configuration": "Base models: GPT Claude Grok Gemini DeepSeek", "score": 0.9401993355481728, "raw_score": 0.9401993355481728, "raw_scale": 1, "source_record": "Shadow Harness V8.01|GPT Claude Grok Gemini DeepSeek|2026-10-12", "source_date": "2026-10-12", "reported_by": "Shadow" }, { "id": "15dd55c73874cd2b86fc", "model": "CustomGPT.ai Research Lab v44", "organization": "CustomGPT.ai", "configuration": "Base models: CustomGPT.ai enterprise agent, Claude, Gemini, GPT", "score": 0.9335548172757475, "raw_score": 0.9335548172757475, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v44|CustomGPT.ai enterprise agent, Claude, Gemini, GPT|2026-06-03", "source_date": "2026-06-03", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-agent" }, { "id": "db1f76c6ed598406da83", "model": "OmniSeek_Preview_v1.1", "organization": "Alibaba Enterprise Intelligence & HKUST(GZ)", "configuration": "Base models: Qwen3.8 Max, GPT-5.5, Gemini3.7 Flash", "score": 0.9335548172757475, "raw_score": 0.9335548172757475, "raw_scale": 1, "source_record": "OmniSeek_Preview_v1.1|Qwen3.8 Max, GPT-5.5, Gemini3.7 Flash|2026-10-30", "source_date": "2026-10-30", "reported_by": "Alibaba Enterprise Intelligence & HKUST(GZ)" }, { "id": "9351cfc696125ac075fe", "model": "Co-Sight Pro v1.0.1", "organization": "ZTE-AICloud", "configuration": "Base models: ZTE Nebula LLM, Gemini 3.1 Pro, GPT 5.5, Claude Opus 4.7", "score": 0.9302325581395349, "raw_score": 0.9302325581395349, "raw_scale": 1, "source_record": "Co-Sight Pro v1.0.1|ZTE Nebula LLM, Gemini 3.1 Pro, GPT 5.5, Claude Opus 4.7|2026-05-16", "source_date": "2026-05-16", "reported_by": "ZTE-AICloud", "report_url": "https://github.com/ZTE-AICloud/Co-Sight" }, { "id": "e2491582429c40c9fc7c", "model": "Keen AgenticOS", "organization": "KeenData", "configuration": "Base models: Deeepseek ，Kimi，Claude", "score": 0.9302325581395349, "raw_score": 0.9302325581395349, "raw_scale": 1, "source_record": "Keen AgenticOS|Deeepseek ，Kimi，Claude|2026-10-29", "source_date": "2026-10-29", "reported_by": "KeenData" }, { "id": "af9c8d34225f0539219a", "model": "Shadow Harness V2.01", "organization": "Shadow", "configuration": "Base models: GPT Claude Grok Gemini DeepSeek", "score": 0.9302325581395349, "raw_score": 0.9302325581395349, "raw_scale": 1, "source_record": "Shadow Harness V2.01|GPT Claude Grok Gemini DeepSeek|2026-10-10", "source_date": "2026-10-10", "reported_by": "Shadow", "report_url": "https://rysx.com.cn/" }, { "id": "a9eb9d914a5d783cf6d0", "model": "Shadow Harness V3.01", "organization": "Shadow", "configuration": "Base models: GPT Claude Grok Gemini DeepSeek", "score": 0.9302325581395349, "raw_score": 0.9302325581395349, "raw_scale": 1, "source_record": "Shadow Harness V3.01|GPT Claude Grok Gemini DeepSeek|2026-10-11", "source_date": "2026-10-11", "reported_by": "Shadow" }, { "id": "cfe01693e89094540813", "model": "Shadow Harness V4.01", "organization": "Shadow", "configuration": "Base models: GPT Claude Gemini DeepSeek", "score": 0.9302325581395349, "raw_score": 0.9302325581395349, "raw_scale": 1, "source_record": "Shadow Harness V4.01|GPT Claude Gemini DeepSeek|2026-10-11", "source_date": "2026-10-11", "reported_by": "Shadow" }, { "id": "119ff70acc63817fc395", "model": "Shadow Harness V5.01", "organization": "Shadow", "configuration": "Base models: GPT Claude Gemini DeepSeek Grok", "score": 0.9302325581395349, "raw_score": 0.9302325581395349, "raw_scale": 1, "source_record": "Shadow Harness V5.01|GPT Claude Gemini DeepSeek Grok|2026-10-11", "source_date": "2026-10-11", "reported_by": "Shadow" }, { "id": "4c65ad50fdfad6b1dc6e", "model": "Shadow Harness V6.01", "organization": "Shadow", "configuration": "Base models: GPT Claude Gemini DeepSeek Grok", "score": 0.9302325581395349, "raw_score": 0.9302325581395349, "raw_scale": 1, "source_record": "Shadow Harness V6.01|GPT Claude Gemini DeepSeek Grok|2026-10-12", "source_date": "2026-10-12", "reported_by": "Shadow" }, { "id": "ca1bae2aa939a196439b", "model": "Shadow Harness V7.01", "organization": "Shadow", "configuration": "Base models: GPT Claude Grok Gemini DeepSeek", "score": 0.9302325581395349, "raw_score": 0.9302325581395349, "raw_scale": 1, "source_record": "Shadow Harness V7.01|GPT Claude Grok Gemini DeepSeek|2026-10-12", "source_date": "2026-10-12", "reported_by": "Shadow" }, { "id": "2243c772c8f9c53683ce", "model": "Shadow Harness V 1.04", "organization": "Shadow", "configuration": "Base models: GPT Claude Gemini DeepSeek", "score": 0.9269102990033222, "raw_score": 0.9269102990033222, "raw_scale": 1, "source_record": "Shadow Harness V 1.04|GPT Claude Gemini DeepSeek|2026-10-08", "source_date": "2026-10-08", "reported_by": "Shadow" }, { "id": "5f2d4be9847cc06a8b07", "model": "Shadow Harness V1.05", "organization": "Shadow", "configuration": "Base models: GPT Claude Grok Gemini DeepSeek", "score": 0.9269102990033222, "raw_score": 0.9269102990033222, "raw_scale": 1, "source_record": "Shadow Harness V1.05|GPT Claude Grok Gemini DeepSeek|2026-10-08", "source_date": "2026-10-08", "reported_by": "Shadow" }, { "id": "7d24ef2d5ee045ddd496", "model": "Shadow Harness V1.07", "organization": "Shadow", "configuration": "Base models: Grok GPT Claude DeepSeek", "score": 0.9269102990033222, "raw_score": 0.9269102990033222, "raw_scale": 1, "source_record": "Shadow Harness V1.07|Grok GPT Claude DeepSeek|2026-10-09", "source_date": "2026-10-09", "reported_by": "Shadow" }, { "id": "07f73ebb133c4673cfa5", "model": "OPS-Agentic-Search", "organization": "Alibaba Cloud", "configuration": "Base models: Qwen, Claude 4.6, GPT 5, DeepSeek 3.2, Gemini 3 Pro, Kimi K2.5", "score": 0.9235880398671097, "raw_score": 0.9235880398671097, "raw_scale": 1, "source_record": "OPS-Agentic-Search|Qwen, Claude 4.6, GPT 5, DeepSeek 3.2, Gemini 3 Pro, Kimi K2.5|2026-03-11", "source_date": "2026-03-11", "reported_by": "Alibaba Cloud", "report_url": "https://www.alibabacloud.com/en/product/opensearch?_p_lc=1" }, { "id": "2ec907beed2ad5ab2e44", "model": "SHAPARTNER V1.01", "organization": "SHAPARTNER", "configuration": "Base models: GPT Claude Gemini DeepSeek", "score": 0.9235880398671097, "raw_score": 0.9235880398671097, "raw_scale": 1, "source_record": "SHAPARTNER V1.01|GPT Claude Gemini DeepSeek|2026-10-10", "source_date": "2026-10-10", "reported_by": "SHAPARTNER" }, { "id": "394ebd9621f2e5f17728", "model": "SHAPARTNER V1.02", "organization": "SHAPARTNER", "configuration": "Base models: GPT Claude Gemini DeepSeek Grok", "score": 0.9235880398671097, "raw_score": 0.9235880398671097, "raw_scale": 1, "source_record": "SHAPARTNER V1.02|GPT Claude Gemini DeepSeek Grok|2026-10-10", "source_date": "2026-10-10", "reported_by": "SHAPARTNER" }, { "id": "028164168001ee6ed17a", "model": "Shadow Harness V14", "organization": "Shadow", "configuration": "Base models: GPT Claude Grok", "score": 0.9235880398671097, "raw_score": 0.9235880398671097, "raw_scale": 1, "source_record": "Shadow Harness V14|GPT Claude Grok|2026-09-22", "source_date": "2026-09-22", "reported_by": "Shadow", "report_url": "https://rysx.com.cn/" }, { "id": "55c147ce6c5ddaa959e6", "model": "AuraAi", "organization": "AI", "configuration": "Base models: Claude", "score": 0.920265780730897, "raw_score": 0.920265780730897, "raw_scale": 1, "source_record": "AuraAi|Claude|2026-05-27", "source_date": "2026-05-27", "reported_by": "AI" }, { "id": "c30b7ff6cab8bf8db37a", "model": "Co-Sight Pro v1.0.0", "organization": "ZTE-AICloud", "configuration": "Base models: ZTE Nebula LLM, claude-opus-4.7, gemini-3.1-pro-preview", "score": 0.920265780730897, "raw_score": 0.920265780730897, "raw_scale": 1, "source_record": "Co-Sight Pro v1.0.0|ZTE Nebula LLM, claude-opus-4.7, gemini-3.1-pro-preview|2026-05-11", "source_date": "2026-05-11", "reported_by": "ZTE-AICloud", "report_url": "https://github.com/ZTE-AICloud/Co-Sight" }, { "id": "388951089e6a1d0e951a", "model": "Cobra Agent Orch", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5, Gemini 3 Pro, Claude Opus 4.8", "score": 0.920265780730897, "raw_score": 0.920265780730897, "raw_scale": 1, "source_record": "Cobra Agent Orch|GPT 5.5, Gemini 3 Pro, Claude Opus 4.8|2026-06-23", "source_date": "2026-06-23", "reported_by": "Dalpha Inc" }, { "id": "8dae5532a214b762821f", "model": "CustomGPT.ai Research Lab V42", "organization": "CustomGPT.ai", "configuration": "Base models: CustomGPT.ai Enterprise Agents", "score": 0.920265780730897, "raw_score": 0.920265780730897, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab V42|CustomGPT.ai Enterprise Agents|2026-05-22", "source_date": "2026-05-22", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-agent" }, { "id": "b5ac307b3141a2d2b462", "model": "CustomGPT.ai Research Lab v41", "organization": "CustomGPT.ai", "configuration": "Base models: Claude, GPT, Gemini", "score": 0.920265780730897, "raw_score": 0.920265780730897, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v41|Claude, GPT, Gemini|2026-05-22", "source_date": "2026-05-22", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-agent" }, { "id": "2a71aba8eda93d22a1c9", "model": "CustomGPT.ai Research Lab v43", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.920265780730897, "raw_score": 0.920265780730897, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v43|Claude|2026-05-23", "source_date": "2026-05-23", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-manus-agent" }, { "id": "b9a59dc139dbcb01737b", "model": "OmniSeek_Preview_v1", "organization": "Alibaba Enterprise Intelligence & HKUST(GZ)", "configuration": "Base models: Qwen3.8 Max, GPT-5.5, Gemini3.7 Flash", "score": 0.920265780730897, "raw_score": 0.920265780730897, "raw_scale": 1, "source_record": "OmniSeek_Preview_v1|Qwen3.8 Max, GPT-5.5, Gemini3.7 Flash|2026-10-23", "source_date": "2026-10-23", "reported_by": "Alibaba Enterprise Intelligence & HKUST(GZ)" }, { "id": "cfa03110e5ccea56cd10", "model": "SHAPARTNER V11", "organization": "SHAPARTNER", "configuration": "Base models: GPT Claude Gemini DeepSeek", "score": 0.920265780730897, "raw_score": 0.920265780730897, "raw_scale": 1, "source_record": "SHAPARTNER V11|GPT Claude Gemini DeepSeek|2026-09-22", "source_date": "2026-09-22", "reported_by": "SHAPARTNER" }, { "id": "7b22e59c3d8106faa415", "model": "Shadow Harness V13", "organization": "Shadow Harness", "configuration": "Base models: Claude GPT Grok", "score": 0.920265780730897, "raw_score": 0.920265780730897, "raw_scale": 1, "source_record": "Shadow Harness V13|Claude GPT Grok|2026-09-21", "source_date": "2026-09-21", "reported_by": "Shadow Harness", "report_url": "https://rysx.com.cn/" }, { "id": "612ecb59ea152ec3c862", "model": "Cobra Agent_v3.1", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Cobra Agent_v3.1|GPT 5.5|2026-06-23", "source_date": "2026-06-23", "reported_by": "Dalpha Inc" }, { "id": "24eddd317b8495d50dd5", "model": "Cobra Agent_v3.2", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Cobra Agent_v3.2|GPT 5.5|2026-06-23", "source_date": "2026-06-23", "reported_by": "Dalpha Inc" }, { "id": "1337c423e99fb2e7b44a", "model": "Cobra Agent_v3.2.1", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Cobra Agent_v3.2.1|GPT 5.5|2026-06-23", "source_date": "2026-06-23", "reported_by": "Dalpha Inc" }, { "id": "415dc29ddf81374b7337", "model": "Cobra Agent_v3.2.2", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Cobra Agent_v3.2.2|GPT 5.5|2026-06-23", "source_date": "2026-06-23", "reported_by": "Dalpha Inc" }, { "id": "0ba92e3d61ad04dd0170", "model": "Cobra Agent_v3.2.3", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Cobra Agent_v3.2.3|GPT 5.5|2026-06-23", "source_date": "2026-06-23", "reported_by": "Dalpha Inc" }, { "id": "647de117e41a5988ceab", "model": "Cobra Agent_v3.3", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Cobra Agent_v3.3|GPT 5.5|2026-06-23", "source_date": "2026-06-23", "reported_by": "Dalpha Inc" }, { "id": "17940e891e32a5f595e2", "model": "CustomGPT.ai Research Lab v39", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v39|Claude|2026-05-21", "source_date": "2026-05-21", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-agent" }, { "id": "c5ba4505a08439318cc1", "model": "Shadow Harness V1.03", "organization": "Shadow", "configuration": "Base models: Grok GPT", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Shadow Harness V1.03|Grok GPT|2026-10-07", "source_date": "2026-10-07", "reported_by": "Shadow" }, { "id": "721fbe8cadc43e12a784", "model": "Shadow Harness V1.06", "organization": "Shadow", "configuration": "Base models: GPT Claude Gemini DeepSeek", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Shadow Harness V1.06|GPT Claude Gemini DeepSeek|2026-10-08", "source_date": "2026-10-08", "reported_by": "Shadow" }, { "id": "35294e955eaaa3cda347", "model": "Shadow Harness V1.09", "organization": "Shadow", "configuration": "Base models: GPT Claude Gemini DeepSeek Grok", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Shadow Harness V1.09|GPT Claude Gemini DeepSeek Grok|2026-10-09", "source_date": "2026-10-09", "reported_by": "Shadow" }, { "id": "41876f225e988a53a3db", "model": "Shadow Harness V11", "organization": "Shadow", "configuration": "Base models: Claude GPT Grok", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Shadow Harness V11|Claude GPT Grok|2026-09-19", "source_date": "2026-09-19", "reported_by": "Shadow", "report_url": "https://rysx.com.cn/" }, { "id": "ae77092b90c9f01cb30e", "model": "Shadow Harness V15", "organization": "Shadow", "configuration": "Base models: Claude GPT Gemini", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "Shadow Harness V15|Claude GPT Gemini|2026-09-23", "source_date": "2026-09-23", "reported_by": "Shadow" }, { "id": "7dbed0ff5b728ebaed0e", "model": "openJiuwen-deepagent", "organization": "openJiuwen", "configuration": "Base models: GPT5, Gemini-3-Pro", "score": 0.9169435215946844, "raw_score": 0.9169435215946844, "raw_scale": 1, "source_record": "openJiuwen-deepagent|GPT5, Gemini-3-Pro|2026-02-09", "source_date": "2026-02-09", "reported_by": "openJiuwen", "report_url": "https://gitcode.com/openJiuwen/agent-store/tree/main/community/deepagent" }, { "id": "2ead68db1f41397830eb", "model": "Cobra Agent_2.7", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9136212624584718, "raw_score": 0.9136212624584718, "raw_scale": 1, "source_record": "Cobra Agent_2.7|GPT 5.5|2026-06-19", "source_date": "2026-06-19", "reported_by": "Dalpha Inc" }, { "id": "4df21a21cbab38b5c2c9", "model": "Cobra Agent_v2.7", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9136212624584718, "raw_score": 0.9136212624584718, "raw_scale": 1, "source_record": "Cobra Agent_v2.7|GPT 5.5|2026-06-22", "source_date": "2026-06-22", "reported_by": "Dalpha Inc" }, { "id": "a6aabeefe04b26e20f2f", "model": "Cobra Agent_v3.0", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9136212624584718, "raw_score": 0.9136212624584718, "raw_scale": 1, "source_record": "Cobra Agent_v3.0|GPT 5.5|2026-06-22", "source_date": "2026-06-22", "reported_by": "Dalpha Inc" }, { "id": "a90e1aeddd427498b2c4", "model": "Cobra Agent_v3.4", "organization": "Dalpha Inc", "configuration": "Base models: GPT 5.5", "score": 0.9136212624584718, "raw_score": 0.9136212624584718, "raw_scale": 1, "source_record": "Cobra Agent_v3.4|GPT 5.5|2026-06-23", "source_date": "2026-06-23", "reported_by": "Dalpha Inc" }, { "id": "88521d9819363bd7d600", "model": "CustomGPT.ai Research Lab v34", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.9136212624584718, "raw_score": 0.9136212624584718, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v34|Claude|2026-05-19", "source_date": "2026-05-19", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-agent" }, { "id": "cc302c9a679dc6a97c73", "model": "CustomGPT.ai Research Lab v35", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.9136212624584718, "raw_score": 0.9136212624584718, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v35|Claude|2026-05-20", "source_date": "2026-05-20", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-agent" }, { "id": "08daffcdd629ac5d14b8", "model": "Lemon", "organization": "LR AILab of Lenovo CTO Org", "configuration": "Base models: GPT5, Gemini-3-pro, o3", "score": 0.9136212624584718, "raw_score": 0.9136212624584718, "raw_scale": 1, "source_record": "Lemon|GPT5, Gemini-3-pro, o3|2026-02-06", "source_date": "2026-02-06", "reported_by": "LR AILab of Lenovo CTO Org", "report_url": "https://github.com/Open-Lemon/LemonAgent" }, { "id": "d19efc0853f026e7d416", "model": "Shadow Harness V1.02", "organization": "Shadow", "configuration": "Base models: Claude GPT", "score": 0.9136212624584718, "raw_score": 0.9136212624584718, "raw_scale": 1, "source_record": "Shadow Harness V1.02|Claude GPT|2026-10-07", "source_date": "2026-10-07", "reported_by": "Shadow" }, { "id": "117ae027fc9fb57589ce", "model": "CustomGPT.ai Research Lab v33", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.9102990033222591, "raw_score": 0.9102990033222591, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v33|Claude|2026-05-18", "source_date": "2026-05-18", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-agent" }, { "id": "ed709b80ff134dcd063e", "model": "Harness V8", "organization": "Harness", "configuration": "Base models: Grok", "score": 0.9102990033222591, "raw_score": 0.9102990033222591, "raw_scale": 1, "source_record": "Harness V8|Grok|2026-09-18", "source_date": "2026-09-18", "reported_by": "Harness" }, { "id": "4d134e199f6546b43bb6", "model": "SHADOW HARNESS V10", "organization": "SHADOW", "configuration": "Base models: Claude GPT Grok", "score": 0.9102990033222591, "raw_score": 0.9102990033222591, "raw_scale": 1, "source_record": "SHADOW HARNESS V10|Claude GPT Grok|2026-09-19", "source_date": "2026-09-19", "reported_by": "SHADOW" }, { "id": "7b950d9f68cac5a26ed5", "model": "Shadow Harness V1.08", "organization": "Shadow", "configuration": "Base models: GPT Claude Grok Gemini DeepSeek", "score": 0.9102990033222591, "raw_score": 0.9102990033222591, "raw_scale": 1, "source_record": "Shadow Harness V1.08|GPT Claude Grok Gemini DeepSeek|2026-10-09", "source_date": "2026-10-09", "reported_by": "Shadow" }, { "id": "12a18ba55bcfae6c50bf", "model": "Shadow Harness V12", "organization": "Shadow Harness", "configuration": "Base models: GPT Claude Grok", "score": 0.9102990033222591, "raw_score": 0.9102990033222591, "raw_scale": 1, "source_record": "Shadow Harness V12|GPT Claude Grok|2026-09-21", "source_date": "2026-09-21", "reported_by": "Shadow Harness", "report_url": "https://rysx.com.cn" }, { "id": "8e67a498a0aa9a46199c", "model": "Shadow Harness V9", "organization": "Shadow", "configuration": "Base models: Claude GPT Grok", "score": 0.9102990033222591, "raw_score": 0.9102990033222591, "raw_scale": 1, "source_record": "Shadow Harness V9|Claude GPT Grok|2026-09-18", "source_date": "2026-09-18", "reported_by": "Shadow" }, { "id": "991e7543d8213133c3ee", "model": "Shapartner V6", "organization": "Shapartne", "configuration": "Base models: Grok Claude", "score": 0.9102990033222591, "raw_score": 0.9102990033222591, "raw_scale": 1, "source_record": "Shapartner V6|Grok Claude |2026-09-19", "source_date": "2026-09-19", "reported_by": "Shapartne" }, { "id": "1d34d6ec5b5783425040", "model": "CustomGPT.ai Research Lab v31", "organization": "CustomGPT.ai", "configuration": "Base models: claude, gemini, gpt", "score": 0.9069767441860465, "raw_score": 0.9069767441860465, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v31|claude, gemini, gpt|2026-05-15", "source_date": "2026-05-15", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-agent" }, { "id": "b3411d146c3c34bc41a3", "model": "CustomGPT.ai Research Lab v32", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.9069767441860465, "raw_score": 0.9069767441860465, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v32|Claude|2026-05-18", "source_date": "2026-05-18", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/adorosario/customgpt-agent" }, { "id": "a34a000a712186bed863", "model": "Nemotron-ToolOrchestra-0106", "organization": "NVIDIA", "configuration": "Base models: Nemotron-ToolOrchestrator-8B, GPT-5, Claude Opus 4.1, Qwen2.5-Math-72B", "score": 0.9036544850498339, "raw_score": 0.9036544850498339, "raw_scale": 1, "source_record": "Nemotron-ToolOrchestra-0106|Nemotron-ToolOrchestrator-8B, GPT-5, Claude Opus 4.1, Qwen2.5-Math-72B|2026-01-06", "source_date": "2026-01-06", "reported_by": "NVIDIA", "report_url": "https://research.nvidia.com/labs/lpr/ToolOrchestra/" }, { "id": "a9d97b79b1c81ffdb116", "model": "Nemotron-ToolOrchestra-0107", "organization": "NVIDIA", "configuration": "Base models: Nemotron-ToolOrchestrator-8B, GPT-5, Claude Opus 4.1, Qwen2.5-Math-72B", "score": 0.9036544850498339, "raw_score": 0.9036544850498339, "raw_scale": 1, "source_record": "Nemotron-ToolOrchestra-0107|Nemotron-ToolOrchestrator-8B, GPT-5, Claude Opus 4.1, Qwen2.5-Math-72B|2026-01-06", "source_date": "2026-01-06", "reported_by": "NVIDIA", "report_url": "https://research.nvidia.com/labs/lpr/ToolOrchestra/" }, { "id": "ee4d5053d9288ef8d950", "model": "CustomGPT.ai Research Lab v27", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.9003322259136213, "raw_score": 0.9003322259136213, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v27|Claude|2026-05-14", "source_date": "2026-05-14", "reported_by": "CustomGPT.ai", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "16787a915cb71ec87622", "model": "CustomGPT.ai Research Lab v25", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.8970099667774086, "raw_score": 0.8970099667774086, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v25|Claude|2026-05-14", "source_date": "2026-05-14", "reported_by": "CustomGPT.ai", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "259ee0b17bb577a462e1", "model": "CustomGPT.ai Research Lab v26", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.893687707641196, "raw_score": 0.893687707641196, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v26|Claude|2026-05-14", "source_date": "2026-05-14", "reported_by": "CustomGPT.ai", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "3cb09bc9ac89b82bee79", "model": "CustomGPT.ai Research Lab v19", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.8903654485049833, "raw_score": 0.8903654485049833, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v19|Claude|2026-05-12", "source_date": "2026-05-12", "reported_by": "CustomGPT.ai", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "a656930a7e55b01529b6", "model": "CustomGPT.ai Research Lab v16", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.8837209302325582, "raw_score": 0.8837209302325582, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v16|Claude|2026-05-10", "source_date": "2026-05-10", "reported_by": "CustomGPT.ai", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "02e3a690d1e9ef40eea0", "model": "Lemon Agent v1.0.8", "organization": "LR AILab of Lenovo CTO Org", "configuration": "Base models: GPT5, o3, Gemini-3-pro", "score": 0.8837209302325582, "raw_score": 0.8837209302325582, "raw_scale": 1, "source_record": "Lemon Agent v1.0.8|GPT5, o3, Gemini-3-pro|2025-12-04", "source_date": "2025-12-04", "reported_by": "LR AILab of Lenovo CTO Org", "report_url": "https://github.com/Open-Lemon/LemonAgent" }, { "id": "3a3cf33718c73b688d90", "model": "CustomGPT.ai Research Lab v17", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.8803986710963455, "raw_score": 0.8803986710963455, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v17|Claude|2026-05-11", "source_date": "2026-05-11", "reported_by": "CustomGPT.ai", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "c533246471f827f0c69b", "model": "Nemotron-ToolOrchestra", "organization": "NVIDIA", "configuration": "Base models: Nemotron-ToolOrchestrator-8B, GPT-5, Claude Opus 4.1, Qwen2.5-Math-72B", "score": 0.8737541528239202, "raw_score": 0.8737541528239202, "raw_scale": 1, "source_record": "Nemotron-ToolOrchestra|Nemotron-ToolOrchestrator-8B, GPT-5, Claude Opus 4.1, Qwen2.5-Math-72B|2025-12-02", "source_date": "2025-12-02", "reported_by": "NVIDIA", "report_url": "https://research.nvidia.com/labs/lpr/ToolOrchestra/" }, { "id": "019c27f302669a23a805", "model": "Co-Sight_v2.1.0", "organization": "ZTE-AICloud", "configuration": "Base models: ZTE Nebula LLM, Claude Sonnet 4, Gemini 2.5 Pro", "score": 0.8704318936877077, "raw_score": 0.8704318936877077, "raw_scale": 1, "source_record": "Co-Sight_v2.1.0|ZTE Nebula LLM, Claude Sonnet 4, Gemini 2.5 Pro|2025-10-13", "source_date": "2025-10-13", "reported_by": "ZTE-AICloud", "report_url": "https://github.com/ZTE-AICloud/Co-Sight" }, { "id": "d4e0390717e2efc9cbb6", "model": "Favonis Agent", "organization": "Favonis", "configuration": "Base models: GPT 6.1 Sol", "score": 0.8571428571428571, "raw_score": 0.8571428571428571, "raw_scale": 1, "source_record": "Favonis Agent|GPT 6.1 Sol|2026-10-18", "source_date": "2026-10-18", "reported_by": "Favonis", "report_url": "https://favon.is/" }, { "id": "3d2da6dd95fd18a33428", "model": "Keen-Agent-v3", "organization": "Individual", "configuration": "Base models: Kimi + Claude + GPT", "score": 0.8571428571428571, "raw_score": 0.8571428571428571, "raw_scale": 1, "source_record": "Keen-Agent-v3|Kimi + Claude + GPT|2026-10-07", "source_date": "2026-10-07", "reported_by": "Individual", "report_url": "https://github.com/qingguang0309/MiroFlow_GAIA_Keendata" }, { "id": "64e1ab19a9f54e06e984", "model": "Favonis", "organization": "Favonis", "configuration": "Base models: GPT 6.1 Sol", "score": 0.8504983388704319, "raw_score": 0.8504983388704319, "raw_scale": 1, "source_record": "Favonis|GPT 6.1 Sol|2026-10-19", "source_date": "2026-10-19", "reported_by": "Favonis", "report_url": "https://favon.is/" }, { "id": "655822ff8449b1d07379", "model": "Raccoon AI", "organization": "Raccoon AI", "configuration": "Base models: Gemini 3 Pro, Claude Opus 4.6, GPT 5.1", "score": 0.8504983388704319, "raw_score": 0.8504983388704319, "raw_scale": 1, "source_record": "Raccoon AI|Gemini 3 Pro, Claude Opus 4.6, GPT 5.1|2026-06-20", "source_date": "2026-06-20", "reported_by": "Raccoon AI", "report_url": "https://raccoonai.tech/" }, { "id": "74cb8e7dc2e112072f48", "model": "Keen-Agent-v4", "organization": "Individual", "configuration": "Base models: Kimi + Claude + GPT", "score": 0.8471760797342193, "raw_score": 0.8471760797342193, "raw_scale": 1, "source_record": "Keen-Agent-v4|Kimi + Claude + GPT|2026-10-29", "source_date": "2026-10-29", "reported_by": "Individual", "report_url": "https://github.com/qingguang0309/MiroFlow_GAIA_Keendata" }, { "id": "f17bbc332ad5fd922dab", "model": "Mimir v0.9.13", "organization": "Team Norse", "configuration": "Base models: GPT, Gemini, Claude", "score": 0.8471760797342193, "raw_score": 0.8471760797342193, "raw_scale": 1, "source_record": "Mimir v0.9.13|GPT, Gemini, Claude|2026-03-06", "source_date": "2026-03-06", "reported_by": "Team Norse", "report_url": "https://mimir-paper.vercel.app/" }, { "id": "6b72ec734c422251a3a2", "model": "Co-Sight_v2.0.1", "organization": "ZTE-AICloud", "configuration": "Base models: ZTE Nebula LLM, Claude Sonnet 4, Gemini 2.5 Pro", "score": 0.8438538205980066, "raw_score": 0.8438538205980066, "raw_scale": 1, "source_record": "Co-Sight_v2.0.1|ZTE Nebula LLM, Claude Sonnet 4, Gemini 2.5 Pro|2025-09-24", "source_date": "2025-09-24", "reported_by": "ZTE-AICloud", "report_url": "https://github.com/ZTE-AICloud/Co-Sight" }, { "id": "3771a5912dc1343d9dcd", "model": "CustomGPT.ai Research Lab v11", "organization": "CustomGPT.ai Research Lab", "configuration": "Base models: Claude", "score": 0.8438538205980066, "raw_score": 0.8438538205980066, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v11|Claude|2026-05-07", "source_date": "2026-05-07", "reported_by": "CustomGPT.ai Research Lab", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "0630208a3114cfb9102a", "model": "Co-Sight_v2.0.0", "organization": "ZTE-AICloud", "configuration": "Base models: Claude Sonnet 4, Gemini 2.5 Pro", "score": 0.840531561461794, "raw_score": 0.840531561461794, "raw_scale": 1, "source_record": "Co-Sight_v2.0.0|Claude Sonnet 4, Gemini 2.5 Pro|2025-08-30", "source_date": "2025-08-30", "reported_by": "ZTE-AICloud", "report_url": "https://github.com/ZTE-AICloud/Co-Sight" }, { "id": "265ce0c13b322b7807bd", "model": "h2oGPTe Agent v1.6.44", "organization": "h2o.ai", "configuration": "Base models: sonnet-4-5 + gpt-5", "score": 0.8372093023255814, "raw_score": 0.8372093023255814, "raw_scale": 1, "source_record": "h2oGPTe Agent v1.6.44|sonnet-4-5 + gpt-5|2025-11-03", "source_date": "2025-11-03", "reported_by": "h2o.ai", "report_url": "https://h2ogpte.genai.h2o.ai/" }, { "id": "8757506cdde8bf8a5595", "model": "CustomGPT.ai Research Lab v5", "organization": "CustomGPT.ai", "configuration": "Base models: Claude", "score": 0.8338870431893688, "raw_score": 0.8338870431893688, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v5|Claude|2026-05-06", "source_date": "2026-05-06", "reported_by": "CustomGPT.ai", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "24b7cbd33dd33f965047", "model": "Skywork Deep Research Agent v2", "organization": "Skywork AI", "configuration": "Base models: skywork-agent-model, gpt-4.1, claude-3.7-sonnet, gemini-2.5-pro", "score": 0.8338870431893688, "raw_score": 0.8338870431893688, "raw_scale": 1, "source_record": "Skywork Deep Research Agent v2|skywork-agent-model, gpt-4.1, claude-3.7-sonnet, gemini-2.5-pro|2025-08-13", "source_date": "2025-08-13", "reported_by": "Skywork AI", "report_url": "https://skyworkai.github.io/DeepResearchAgent" }, { "id": "ad56e3693af5b7364a74", "model": "CustomGPT.ai Research Lab v2", "organization": "CustomGPT.ai Research Lab", "configuration": "Base models: Claude", "score": 0.8305647840531561, "raw_score": 0.8305647840531561, "raw_scale": 1, "source_record": "CustomGPT.ai Research Lab v2|Claude|2026-05-05", "source_date": "2026-05-05", "reported_by": "CustomGPT.ai Research Lab", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "31f987f522ebf881fd99", "model": "Keen-Agent-v2", "organization": "Individual", "configuration": "Base models: Kimi + Claude + GPT", "score": 0.8205980066445183, "raw_score": 0.8205980066445183, "raw_scale": 1, "source_record": "Keen-Agent-v2|Kimi + Claude + GPT|2026-09-20", "source_date": "2026-09-20", "reported_by": "Individual", "report_url": "https://github.com/qingguang0309/MiroFlow_GAIA_Keendata" }, { "id": "6075af0399ba77f19962", "model": "🐦‍🔥 AWorld (Run Instantly)", "organization": "inclusionAI", "configuration": "Base models: GPT-4o, DeepSeek V3, Claude-Sonnet-4, Gemini-2.5-Pro", "score": 0.8172757475083057, "raw_score": 0.8172757475083057, "raw_scale": 1, "source_record": "🐦‍🔥 AWorld (Run Instantly)|GPT-4o, DeepSeek V3, Claude-Sonnet-4, Gemini-2.5-Pro|2025-08-06", "source_date": "2025-08-06", "reported_by": "inclusionAI", "report_url": "https://github.com/inclusionAI/AWorld/tree/main" }, { "id": "7e1221c4608f92cc0521", "model": "Mimir", "organization": "Team Norse", "configuration": "Base models: GPT, Gemini, Claude", "score": 0.7973421926910299, "raw_score": 0.7973421926910299, "raw_scale": 1, "source_record": "Mimir|GPT, Gemini, Claude|2026-02-27", "source_date": "2026-02-27", "reported_by": "Team Norse", "report_url": "https://mimir-paper.vercel.app/" }, { "id": "c59e3f88e0a96b97aed9", "model": "h2oGPTe Agent v1.6.33", "organization": "h2o.ai", "configuration": "Base models: claude-3-7-sonnet-20250219, gemini-2.5-pro-preview-06-05 (extended thinking)", "score": 0.7973421926910299, "raw_score": 0.7973421926910299, "raw_scale": 1, "source_record": "h2oGPTe Agent v1.6.33|claude-3-7-sonnet-20250219, gemini-2.5-pro-preview-06-05 (extended thinking)|2025-06-22", "source_date": "2025-06-22", "reported_by": "h2o.ai", "report_url": "https://id.public.h2o.ai/" }, { "id": "d530cde6a98ed6db14e7", "model": "AG-GAIA-S7", "organization": "AG", "configuration": "Base models: Claude", "score": 0.7906976744186046, "raw_score": 0.7906976744186046, "raw_scale": 1, "source_record": "AG-GAIA-S7|Claude|2026-04-21", "source_date": "2026-04-21", "reported_by": "AG", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "0bcf9cfc6beeae4ee45c", "model": "ARI-CLI V11 0322", "organization": "Clevi", "configuration": "Base models: cip-5.5-agent, cip-5.5-mm", "score": 0.7906976744186046, "raw_score": 0.7906976744186046, "raw_scale": 1, "source_record": "ARI-CLI V11 0322|cip-5.5-agent, cip-5.5-mm|2026-03-22", "source_date": "2026-03-22", "reported_by": "Clevi", "report_url": "https://clevi.ai/ko" }, { "id": "99b3b2348748dc64939b", "model": "MiroFlow (Reproducible!)", "organization": "MiroMind AI", "configuration": "Base models: GPT-5 (agent), o3 (hints, summary), Gemini-2.5-Pro (as tool), Claude (as tool)", "score": 0.7906976744186046, "raw_score": 0.7906976744186046, "raw_scale": 1, "source_record": "MiroFlow (Reproducible!)|GPT-5 (agent), o3 (hints, summary), Gemini-2.5-Pro (as tool), Claude (as tool)|2025-09-16", "source_date": "2025-09-16", "reported_by": "MiroMind AI", "report_url": "https://github.com/MiroMindAI/MiroFlow" }, { "id": "fe6d7753510c2b14e73b", "model": "h2oGPTe Agent v1.6.32", "organization": "h2o.ai", "configuration": "Base models: claude-3-7-sonnet-20250219, gemini-2.5-pro-preview-06-05", "score": 0.7906976744186046, "raw_score": 0.7906976744186046, "raw_scale": 1, "source_record": "h2oGPTe Agent v1.6.32|claude-3-7-sonnet-20250219, gemini-2.5-pro-preview-06-05|2025-06-18", "source_date": "2025-06-18", "reported_by": "h2o.ai", "report_url": "https://id.public.h2o.ai/" }, { "id": "c77d9a4f9e6c97be1f94", "model": "abcdagent v26.09.28", "organization": "abcd", "configuration": "Base models: gpt-5.5", "score": 0.7873754152823921, "raw_score": 0.7873754152823921, "raw_scale": 1, "source_record": "abcdagent v26.09.28|gpt-5.5|2026-09-28", "source_date": "2026-09-28", "reported_by": "abcd", "report_url": "https://developers.openai.com/api/docs/models/gpt-5" }, { "id": "b81ca361b2b0b264e209", "model": "ari-cli with arbiter agent", "organization": "CLEVI", "configuration": "Base models: cip-5.5-agent, cip-5.5-mm", "score": 0.7873754152823921, "raw_score": 0.7873754152823921, "raw_scale": 1, "source_record": "ari-cli with arbiter agent|cip-5.5-agent, cip-5.5-mm|2026-03-23", "source_date": "2026-03-23", "reported_by": "CLEVI", "report_url": "https://clevi.ai/ko" }, { "id": "8689f82b1c02f691343c", "model": "🦤 AWorld (Run Instantly)", "organization": "inclusionAI", "configuration": "Base models: GPT-4o, DeepSeek V3, Claude-Sonnet-4, Gemini-2.5-Pro", "score": 0.770764119601329, "raw_score": 0.770764119601329, "raw_scale": 1, "source_record": "🦤 AWorld (Run Instantly)|GPT-4o, DeepSeek V3, Claude-Sonnet-4, Gemini-2.5-Pro|2025-06-26", "source_date": "2025-06-26", "reported_by": "inclusionAI", "report_url": "https://github.com/inclusionAI/AWorld/tree/main" }, { "id": "19ed0c19569e9072f943", "model": "Keen-Agent", "organization": "Individual", "configuration": "Base models: Kimi + Claude + GPT", "score": 0.7674418604651163, "raw_score": 0.7674418604651163, "raw_scale": 1, "source_record": "Keen-Agent|Kimi + Claude + GPT|2026-09-07", "source_date": "2026-09-07", "reported_by": "Individual", "report_url": "https://github.com/qingguang0309/MiroFlow_GAIA_Keendata" }, { "id": "e5c52139e183d4b87250", "model": "remio_gpt5.5", "organization": "everdent.ai", "configuration": "Base models: GPT-5.5", "score": 0.7574750830564784, "raw_score": 0.7574750830564784, "raw_scale": 1, "source_record": "remio_gpt5.5|GPT-5.5|2026-07-23", "source_date": "2026-07-23", "reported_by": "everdent.ai", "report_url": "https://openai.com/zh-Hans-CN/index/introducing-gpt-5-5/" }, { "id": "bcd487359e133c4f36f6", "model": "ARI-Self Drive Without Prompt", "organization": "Clevi", "configuration": "Base models: cip-5.5-agent, cip-5.5-mm", "score": 0.7475083056478405, "raw_score": 0.7475083056478405, "raw_scale": 1, "source_record": "ARI-Self Drive Without Prompt|cip-5.5-agent, cip-5.5-mm|2026-03-19", "source_date": "2026-03-19", "reported_by": "Clevi", "report_url": "https://clevi.ai/" }, { "id": "1c30367a2ded08bdbe28", "model": "h2oGPTe Agent v1.6.27 | March 17 original date", "organization": "h2o.ai", "configuration": "Base models: claude-3-7-sonnet", "score": 0.7475083056478405, "raw_score": 0.7475083056478405, "raw_scale": 1, "source_record": "h2oGPTe Agent v1.6.27 | March 17 original date|claude-3-7-sonnet|2025-03-20", "source_date": "2025-03-20", "reported_by": "h2o.ai", "report_url": "https://h2ogpte.genai.h2o.ai/" }, { "id": "ac9cb00db6d3684e4d7b", "model": "ARI CLI", "organization": "Clevi", "configuration": "Base models: cip-5.5-agent", "score": 0.7441860465116279, "raw_score": 0.7441860465116279, "raw_scale": 1, "source_record": "ARI CLI|cip-5.5-agent|2026-03-21", "source_date": "2026-03-21", "reported_by": "Clevi", "report_url": "https://clevi.ai/ko" }, { "id": "7e2db146f3c18d2599bc", "model": "Agnes AI", "organization": "Sapiens AI", "configuration": "Base models: Agnes AI", "score": 0.7441860465116279, "raw_score": 0.7441860465116279, "raw_scale": 1, "source_record": "Agnes AI|Agnes AI|2026-06-11", "source_date": "2026-06-11", "reported_by": "Sapiens AI", "report_url": "https://agnes-ai.com" }, { "id": "a38771c38a56187027e5", "model": "Clawdbot", "organization": "Nora AI", "configuration": "Base models: Claude Opus 4.5", "score": 0.7408637873754153, "raw_score": 0.7408637873754153, "raw_scale": 1, "source_record": "Clawdbot|Claude Opus 4.5|2026-01-29", "source_date": "2026-01-29", "reported_by": "Nora AI", "report_url": "https://docs.clawd.bot" }, { "id": "c466dd183136e07acf09", "model": "CustomGPT Agent v2.1", "organization": "CustomGPT.ai", "configuration": "Base models: Claude (Opus 4.6 + Sonnet 4.6)", "score": 0.7408637873754153, "raw_score": 0.7408637873754153, "raw_scale": 1, "source_record": "CustomGPT Agent v2.1|Claude (Opus 4.6 + Sonnet 4.6)|2026-03-14", "source_date": "2026-03-14", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/uckmhnds/customgpt-agent" }, { "id": "d77433169101df2e3622", "model": "CustomGPT Agent v2", "organization": "CustomGPT.ai", "configuration": "Base models: Claude (Opus 4.6 + Sonnet 4.6)", "score": 0.7375415282392026, "raw_score": 0.7375415282392026, "raw_scale": 1, "source_record": "CustomGPT Agent v2|Claude (Opus 4.6 + Sonnet 4.6)|2026-03-13", "source_date": "2026-03-13", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/uckmhnds/customgpt-agent" }, { "id": "0722275e0be52f8a773a", "model": "AiDE-Agent_v9", "organization": "ValueLabs", "configuration": "Base models: Claude-sonnet-4-5", "score": 0.7342192691029901, "raw_score": 0.7342192691029901, "raw_scale": 1, "source_record": "AiDE-Agent_v9\n|Claude-sonnet-4-5 \n|2025-11-24", "source_date": "2025-11-24", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "cf1f86cfb6f5a33a3451", "model": "AiDE-Agent_v7", "organization": "ValueLabs", "configuration": "Base models: Claude", "score": 0.7242524916943521, "raw_score": 0.7242524916943521, "raw_scale": 1, "source_record": "AiDE-Agent_v7|Claude|2025-11-19", "source_date": "2025-11-19", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "4f52095b1a62dbe53f82", "model": "🦩 AWorld (Run Instantly)", "organization": "inclusionAI", "configuration": "Base models: GPT-4o, DeepSeek V3, Claude-3.7-Sonnet, Gemini 2.5 Pro", "score": 0.7242524916943521, "raw_score": 0.7242524916943521, "raw_scale": 1, "source_record": "🦩 AWorld (Run Instantly)|GPT-4o, DeepSeek V3, Claude-3.7-Sonnet, Gemini 2.5 Pro|2025-06-19", "source_date": "2025-06-19", "reported_by": "inclusionAI", "report_url": "https://github.com/inclusionAI/AWorld/tree/main" }, { "id": "c51a0db52ad6c0b16b2b", "model": "Corint v1.0", "organization": "Futurity", "configuration": "Base models: DeepSeek V4 Pro", "score": 0.717607973421927, "raw_score": 0.717607973421927, "raw_scale": 1, "source_record": "Corint v1.0|DeepSeek V4 Pro|2026-05-10", "source_date": "2026-05-10", "reported_by": "Futurity", "report_url": "https://futurity.work" }, { "id": "bc8fca2eaa3bd54d4bea", "model": "CustomGPT Agent", "organization": "CustomGPT.ai", "configuration": "Base models: Claude (Opus 4.6 + Sonnet 4.6)", "score": 0.717607973421927, "raw_score": 0.717607973421927, "raw_scale": 1, "source_record": "CustomGPT Agent|Claude (Opus 4.6 + Sonnet 4.6)|2026-02-27", "source_date": "2026-02-27", "reported_by": "CustomGPT.ai", "report_url": "https://github.com/uckmhnds/customgpt-agent" }, { "id": "9b6dde49a0149465f08a", "model": "AiDE-Agent_v5", "organization": "ValueLabs", "configuration": "Base models: Claude", "score": 0.7142857142857143, "raw_score": 0.7142857142857143, "raw_scale": 1, "source_record": "AiDE-Agent_v5|Claude |2025-11-18", "source_date": "2025-11-18", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "3fb0cefeab88d589c882", "model": "Durandal (deepseek-v4-pro)", "organization": "cloudai.nexus", "configuration": "Base models: DeepSeek-V4-Pro", "score": 0.7109634551495017, "raw_score": 0.7109634551495017, "raw_scale": 1, "source_record": "Durandal (deepseek-v4-pro)|DeepSeek-V4-Pro|2026-09-18", "source_date": "2026-09-18", "reported_by": "cloudai.nexus", "report_url": "https://api-docs.deepseek.com" }, { "id": "a0089d39a274e6213b52", "model": "ARI on SVCE with CTA-Subagents", "organization": "Clevi", "configuration": "Base models: cip-5.5-agent, cip-5.5-mm", "score": 0.707641196013289, "raw_score": 0.707641196013289, "raw_scale": 1, "source_record": "ARI on SVCE with CTA-Subagents|cip-5.5-agent, cip-5.5-mm|2026-03-18", "source_date": "2026-03-18", "reported_by": "Clevi", "report_url": "https://clevi.ai/" }, { "id": "73cd8ae5a43c2c8ae758", "model": "AiDE-Agent_v6", "organization": "ValueLabs", "configuration": "Base models: Claude", "score": 0.7043189368770764, "raw_score": 0.7043189368770764, "raw_scale": 1, "source_record": "AiDE-Agent_v6|Claude|2025-11-19", "source_date": "2025-11-19", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "0f0b56324fb55857625a", "model": "Agnes AI", "organization": "Agnes AI", "configuration": "Base models: GPT-5.5", "score": 0.7009966777408638, "raw_score": 0.7009966777408638, "raw_scale": 1, "source_record": "Agnes AI|GPT-5.5|2026-07-14", "source_date": "2026-07-14", "reported_by": "Agnes AI", "report_url": "https://agnes-ai.com/" }, { "id": "c5fa4d1f4f39413d4d8b", "model": "remio", "organization": "everdent.ai", "configuration": "Base models: GPT-5.4", "score": 0.6976744186046512, "raw_score": 0.6976744186046512, "raw_scale": 1, "source_record": "remio|GPT-5.4|2026-07-20", "source_date": "2026-07-20", "reported_by": "everdent.ai", "report_url": "https://openai.com/zh-Hans-CN/index/introducing-gpt-5-4/" }, { "id": "0a748e35a0410370caae", "model": "Sukuna V0.1.5", "organization": "Midsphere AI", "configuration": "Base models: gpt-5, gemini-2.5-pro, o3, claude-sonnet-4", "score": 0.6943521594684385, "raw_score": 0.6943521594684385, "raw_scale": 1, "source_record": "Sukuna V0.1.5|gpt-5, gemini-2.5-pro, o3, claude-sonnet-4|2025-10-06", "source_date": "2025-10-06", "reported_by": "Midsphere AI", "report_url": "https://www.midsphere.ai/" }, { "id": "5ce946b34430bb7d0656", "model": "AiDE-Agent_v4", "organization": "ValueLabs", "configuration": "Base models: Claude", "score": 0.6877076411960132, "raw_score": 0.6877076411960132, "raw_scale": 1, "source_record": "AiDE-Agent_v4|Claude|2025-11-13", "source_date": "2025-11-13", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "2a8179f6a1a91d32a551", "model": "ReAct_ds_fr", "organization": "imperial college london", "configuration": "Base models: DeepSeek v4 flash", "score": 0.6810631229235881, "raw_score": 0.6810631229235881, "raw_scale": 1, "source_record": "ReAct_ds_fr|DeepSeek v4 flash|2026-08-12", "source_date": "2026-08-12", "reported_by": "imperial college london", "report_url": "https://api-docs.deepseek.com/" }, { "id": "1f3b4bac9ec03b0db8a3", "model": "Slai Agent", "organization": "slai.life", "configuration": "Base models: gpt 5.6 terra", "score": 0.6810631229235881, "raw_score": 0.6810631229235881, "raw_scale": 1, "source_record": "Slai Agent|gpt 5.6 terra|2026-10-04", "source_date": "2026-10-04", "reported_by": "slai.life", "report_url": "https://openai.com" }, { "id": "5a920821f5124f1f8528", "model": "AiDE-Agent_v3", "organization": "ValueLabs", "configuration": "Base models: Claude", "score": 0.6710963455149501, "raw_score": 0.6710963455149501, "raw_scale": 1, "source_record": "AiDE-Agent_v3|Claude|2025-11-12", "source_date": "2025-11-12", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "2bf6a02b79fc9d4c71c5", "model": "JoyAgent-Genie-v2", "organization": "JD Enterprise Intelligence (EI)", "configuration": "Base models: OpenAI, Anthropic", "score": 0.6710963455149501, "raw_score": 0.6710963455149501, "raw_scale": 1, "source_record": "JoyAgent-Genie-v2|OpenAI, Anthropic|2025-08-27", "source_date": "2025-08-27", "reported_by": "JD Enterprise Intelligence (EI)", "report_url": "https://github.com/jd-opensource/joyagent-jdgenie" }, { "id": "fe6a0ecfb5adc9f6d578", "model": "OpenHands_Versa", "organization": "CMU, All Hands AI", "configuration": "Base models: OpenAI GPT5", "score": 0.6710963455149501, "raw_score": 0.6710963455149501, "raw_scale": 1, "source_record": "OpenHands_Versa|OpenAI GPT5|2025-10-03", "source_date": "2025-10-03", "reported_by": "CMU, All Hands AI", "report_url": "https://arxiv.org/abs/2506.03011" }, { "id": "3b56b1a8a0af79f347d4", "model": "Slai Agent Runtime", "organization": "slai.life", "configuration": "Base models: GPT-5.6 Terra", "score": 0.6710963455149501, "raw_score": 0.6710963455149501, "raw_scale": 1, "source_record": "Slai Agent Runtime|GPT-5.6 Terra|2026-10-03", "source_date": "2026-10-03", "reported_by": "slai.life", "report_url": "https://openai.com/" }, { "id": "cda4a06e020fbd469e90", "model": "AiDE-Agent_v8", "organization": "ValueLabs", "configuration": "Base models: Claude", "score": 0.6677740863787376, "raw_score": 0.6677740863787376, "raw_scale": 1, "source_record": "AiDE-Agent_v8|Claude|2025-11-24", "source_date": "2025-11-24", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "5746b22febbec431ddd2", "model": "Trase Agent v0.3", "organization": "Trase Systems", "configuration": "Base models: Claude, o1, Gemini", "score": 0.6677740863787376, "raw_score": 0.6677740863787376, "raw_scale": 1, "source_record": "Trase Agent v0.3|Claude, o1, Gemini|2025-02-19", "source_date": "2025-02-19", "reported_by": "Trase Systems", "report_url": "https://trasesystems.com" }, { "id": "504e11687291c36b636a", "model": "AiDE-Agent_v2", "organization": "ValueLabs", "configuration": "Base models: Claude", "score": 0.6644518272425249, "raw_score": 0.6644518272425249, "raw_scale": 1, "source_record": "AiDE-Agent_v2|Claude |2025-11-10", "source_date": "2025-11-10", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "52e18db378a343ce11f0", "model": "Max-1.1", "organization": "Finalmoment", "configuration": "Base models: Deepseek-V4-Flash-4.1, Gpt-Luna, Musespark-1.3", "score": 0.6611295681063123, "raw_score": 0.6611295681063123, "raw_scale": 1, "source_record": "Max-1.1|Deepseek-V4-Flash-4.1, Gpt-Luna, Musespark-1.3|2026-09-16", "source_date": "2026-09-16", "reported_by": "Finalmoment", "report_url": "https://iris.finalmoment.ai/" }, { "id": "3024d5d6b3bea094e158", "model": "OpenHands Versa", "organization": "CMU, All Hands AI", "configuration": "Base models: GPT_5", "score": 0.6578073089700996, "raw_score": 0.6578073089700996, "raw_scale": 1, "source_record": "OpenHands Versa|GPT_5|2025-09-20", "source_date": "2025-09-20", "reported_by": "CMU, All Hands AI", "report_url": "https://github.com/adityasoni9998/OpenHands-Versa" }, { "id": "8d8de7111091d302b93a", "model": "XWork-MultiAgent", "organization": "iFLYTEK", "configuration": "Base models: DeepSeek-V4-Pro", "score": 0.6578073089700996, "raw_score": 0.6578073089700996, "raw_scale": 1, "source_record": "XWork-MultiAgent|DeepSeek-V4-Pro|2026-06-08", "source_date": "2026-06-08", "reported_by": "iFLYTEK", "report_url": "https://api.deepseek.com" }, { "id": "8437ffd2fb58e64bf83b", "model": "EvolAgent", "organization": "KLGR123", "configuration": "Base models: GPT, Claude", "score": 0.6511627906976745, "raw_score": 0.6511627906976745, "raw_scale": 1, "source_record": "EvolAgent|GPT, Claude|2025-07-24", "source_date": "2025-07-24", "reported_by": "KLGR123", "report_url": "https://github.com/KLGR123/EvolAgent" }, { "id": "82a7d4abfd084fa94e08", "model": "JoyAgent-Genie", "organization": "JD Enterprise Intelligence (EI)", "configuration": "Base models: OpenAI, Anthropic", "score": 0.6511627906976745, "raw_score": 0.6511627906976745, "raw_scale": 1, "source_record": "JoyAgent-Genie|OpenAI, Anthropic|2025-07-24", "source_date": "2025-07-24", "reported_by": "JD Enterprise Intelligence (EI)", "report_url": "https://github.com/jd-opensource/joyagent-jdgenie" }, { "id": "9fa9044570f7d05a3b36", "model": "h2oGPTe Agent v1.6.8", "organization": "h2o.ai", "configuration": "Base models: claude-3-5-sonnet", "score": 0.6511627906976745, "raw_score": 0.6511627906976745, "raw_scale": 1, "source_record": "h2oGPTe Agent v1.6.8|claude-3-5-sonnet|2024-12-21", "source_date": "2024-12-21", "reported_by": "h2o.ai", "report_url": "https://h2ogpte.genai.h2o.ai/" }, { "id": "c4acda0e4c85ddd7f146", "model": "Aron", "organization": "FereAI.xyz", "configuration": "Base models: Claude 4.6, Gemini 3.1", "score": 0.6478405315614618, "raw_score": 0.6478405315614618, "raw_scale": 1, "source_record": "Aron|Claude 4.6, Gemini 3.1|2026-03-13", "source_date": "2026-03-13", "reported_by": "FereAI.xyz", "report_url": "https://docs.anthropic.com/en/docs/about-claude/models" }, { "id": "6435fd6c1c633584745c", "model": "Sukuna V0.1.4", "organization": "Midsphere AI", "configuration": "Base models: gpt-5, gemini-2.5-pro, o3, claude-sonnet-4", "score": 0.6478405315614618, "raw_score": 0.6478405315614618, "raw_scale": 1, "source_record": "Sukuna V0.1.4|gpt-5, gemini-2.5-pro, o3, claude-sonnet-4|2025-10-05", "source_date": "2025-10-05", "reported_by": "Midsphere AI", "report_url": "https://midsphere.ai/" }, { "id": "0da70974e563941efcea", "model": "Slai Agent Runtime", "organization": "Slai Agent Runtime", "configuration": "Base models: GPT-5.6 Terra xhigh", "score": 0.6411960132890365, "raw_score": 0.6411960132890365, "raw_scale": 1, "source_record": "Slai Agent Runtime|GPT-5.6 Terra xhigh|2026-10-02", "source_date": "2026-10-02", "reported_by": "Slai Agent Runtime", "report_url": "https://slai.life/" }, { "id": "0751e6871aa99fb2ba85", "model": "🐟 AWorld (Run Instantly)", "organization": "Ant Group", "configuration": "Base models: GPT-4o, DeepSeek V3, Claude-3.7-Sonnet, Gemini 2.5 Pro", "score": 0.6312292358803987, "raw_score": 0.6312292358803987, "raw_scale": 1, "source_record": "🐟 AWorld (Run Instantly)|GPT-4o, DeepSeek V3, Claude-3.7-Sonnet, Gemini 2.5 Pro|2025-05-30", "source_date": "2025-05-30", "reported_by": "Ant Group", "report_url": "https://github.com/inclusionAI/AWorld/tree/main" }, { "id": "da8f44a8eb1e53a210ce", "model": "Evolagent-baseline-v0.1", "organization": "KLGR123", "configuration": "Base models: GPT, Claude", "score": 0.6212624584717608, "raw_score": 0.6212624584717608, "raw_scale": 1, "source_record": "Evolagent-baseline-v0.1|GPT, Claude|2025-07-23", "source_date": "2025-07-23", "reported_by": "KLGR123", "report_url": "https://github.com/KLGR123/EvolAgent" }, { "id": "56a5859cd198940a28c9", "model": "Sukuna V0.1.3", "organization": "Midsphere AI", "configuration": "Base models: gpt-5, gemini-2.5-pro, claude-sonnet-4", "score": 0.6212624584717608, "raw_score": 0.6212624584717608, "raw_scale": 1, "source_record": "Sukuna V0.1.3|gpt-5, gemini-2.5-pro, claude-sonnet-4|2025-10-04", "source_date": "2025-10-04", "reported_by": "Midsphere AI", "report_url": "https://www.midsphere.ai/" }, { "id": "cb61f1d81b7cbfed14a2", "model": "Sukuna V0.1.2", "organization": "Midsphere AI", "configuration": "Base models: gpt-5, gemini-2.5-pro, claude-sonnet-4", "score": 0.5980066445182725, "raw_score": 0.5980066445182725, "raw_scale": 1, "source_record": "Sukuna V0.1.2|gpt-5, gemini-2.5-pro, claude-sonnet-4|2025-10-03", "source_date": "2025-10-03", "reported_by": "Midsphere AI", "report_url": "https://www.midsphere.ai/" }, { "id": "3704b84be4ab1cf90f05", "model": "AiDE-Agent_v1", "organization": "ValueLabs", "configuration": "Base models: Claude & GPT", "score": 0.5913621262458472, "raw_score": 0.5913621262458472, "raw_scale": 1, "source_record": "AiDE-Agent_v1|Claude & GPT|2025-11-07", "source_date": "2025-11-07", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "1b94c338ecf69b4f584b", "model": "Claude Code Ali 2.1.224", "organization": "AgentEval", "configuration": "Base models: deepseek-v4-flash-0731", "score": 0.5880398671096345, "raw_score": 0.5880398671096345, "raw_scale": 1, "source_record": "Claude Code Ali 2.1.224|deepseek-v4-flash-0731|2026-09-03", "source_date": "2026-09-03", "reported_by": "AgentEval", "report_url": "https://help.aliyun.com/zh/model-studio/deepseek-api" }, { "id": "b305ab3a6b5f59aaabd0", "model": "Sukuna V0.1.1", "organization": "Midsphere AI", "configuration": "Base models: gpt-5, gemini-2.5-pro, sonnet-4", "score": 0.5813953488372093, "raw_score": 0.5813953488372093, "raw_scale": 1, "source_record": "Sukuna V0.1.1|gpt-5, gemini-2.5-pro, sonnet-4|2025-10-02", "source_date": "2025-10-02", "reported_by": "Midsphere AI", "report_url": "https://www.midsphere.ai/" }, { "id": "17fe4c97dda7df0d9085", "model": "AiDE-Agent", "organization": "ValueLabs", "configuration": "Base models: Claude & GPT", "score": 0.5614617940199336, "raw_score": 0.5614617940199336, "raw_scale": 1, "source_record": "AiDE-Agent|Claude & GPT|2025-11-06", "source_date": "2025-11-06", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "cbb9962ff34b571cef22", "model": "exo-unoptimized", "organization": "heyalexchoi", "configuration": "Base models: grok 4 fast, gemini 2.5 flash", "score": 0.5548172757475083, "raw_score": 0.5548172757475083, "raw_scale": 1, "source_record": "exo-unoptimized|grok 4 fast, gemini 2.5 flash|2025-11-20", "source_date": "2025-11-20", "reported_by": "heyalexchoi", "report_url": "https://heyalexchoi.com/exo/" }, { "id": "3843ceea07685b330567", "model": "gemini-3.1-pro-s1-react", "organization": "Independent", "configuration": "Base models: gemini-pro-preview-3-1", "score": 0.5548172757475083, "raw_score": 0.5548172757475083, "raw_scale": 1, "source_record": "gemini-3.1-pro-s1-react|gemini-pro-preview-3-1|2026-07-19", "source_date": "2026-07-19", "reported_by": "Independent", "report_url": "https://arxiv.org/abs/2506.08529" }, { "id": "7492c6eaf8a41e857190", "model": "gemini-3.1-pro-s1-react-corrected", "organization": "Independent", "configuration": "Base models: gemini-pro-preview-3-1", "score": 0.5548172757475083, "raw_score": 0.5548172757475083, "raw_scale": 1, "source_record": "gemini-3.1-pro-s1-react-corrected|gemini-pro-preview-3-1|2026-08-10", "source_date": "2026-08-10", "reported_by": "Independent", "report_url": "https://arxiv.org/abs/2506.08529" }, { "id": "590d96d226424a7f4a23", "model": "exo-vanilla-react", "organization": "heyalexchoi", "configuration": "Base models: grok 4 fast, gemini 2.5 flash", "score": 0.5481727574750831, "raw_score": 0.5481727574750831, "raw_scale": 1, "source_record": "exo-vanilla-react|grok 4 fast, gemini 2.5 flash|2025-11-19", "source_date": "2025-11-19", "reported_by": "heyalexchoi", "report_url": "https://heyalexchoi.com/exo/" }, { "id": "b23506ca541c5d6132d4", "model": "AiDE Agents", "organization": "ValueLabs", "configuration": "Base models: claude-sonnet-4-5", "score": 0.5415282392026578, "raw_score": 0.5415282392026578, "raw_scale": 1, "source_record": "AiDE Agents|claude-sonnet-4-5|2025-11-04", "source_date": "2025-11-04", "reported_by": "ValueLabs", "report_url": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "id": "b1a23039008802656899", "model": "Flywork CLI 0.2.4-beta.1 Corrected", "organization": "AgentEval", "configuration": "Base models: Qwen3.8-27B-NVFP4", "score": 0.5415282392026578, "raw_score": 0.5415282392026578, "raw_scale": 1, "source_record": "Flywork CLI 0.2.4-beta.1 Corrected|Qwen3.8-27B-NVFP4|2026-09-01", "source_date": "2026-09-01", "reported_by": "AgentEval", "report_url": "https://qwen.ai/blog?id=qwen3.8" }, { "id": "a84c81b3425675282396", "model": "gemini-3.1-pro-s2-PAR", "organization": "Independent", "configuration": "Base models: gemini-pro-preview-3-1", "score": 0.5415282392026578, "raw_score": 0.5415282392026578, "raw_scale": 1, "source_record": "gemini-3.1-pro-s2-PAR|gemini-pro-preview-3-1|2026-08-12", "source_date": "2026-08-12", "reported_by": "Independent", "report_url": "https://arxiv.org/abs/2506.08529" }, { "id": "fe697e23e6ae1ac209b1", "model": "exo-2025-11-12-distilled-with-feedback", "organization": "heyalexchoi", "configuration": "Base models: grok 4 fast, gemini 2.5 flash", "score": 0.5249169435215947, "raw_score": 0.5249169435215947, "raw_scale": 1, "source_record": "exo-2025-11-12-distilled-with-feedback|grok 4 fast, gemini 2.5 flash|2025-11-21", "source_date": "2025-11-21", "reported_by": "heyalexchoi", "report_url": "https://heyalexchoi.com/exo/" }, { "id": "1b0b7ef17729d22aa921", "model": "Sukuna V0.1", "organization": "Midsphere AI", "configuration": "Base models: gemini-2.5-pro, gpt-5, claude-sonnet-4", "score": 0.5182724252491694, "raw_score": 0.5182724252491694, "raw_scale": 1, "source_record": "Sukuna V0.1|gemini-2.5-pro, gpt-5, claude-sonnet-4|2025-10-01", "source_date": "2025-10-01", "reported_by": "Midsphere AI", "report_url": "https://www.midsphere.ai/" }, { "id": "1b174b42dcc93ce5cc4c", "model": "Ormind v0.2", "organization": "Ormind", "configuration": "Base models: Claude 3.7, GPT4.1, Gemini 2.5", "score": 0.5149501661129569, "raw_score": 0.5149501661129569, "raw_scale": 1, "source_record": "Ormind v0.2|Claude 3.7, GPT4.1, Gemini 2.5|2025-04-20", "source_date": "2025-04-20", "reported_by": "Ormind", "report_url": "https://ormind.ai" }, { "id": "6cce4966f319ba07cd84", "model": "Federal-KB FedQES (GPT-5.6 Luna)", "organization": "Federal-KB", "configuration": "Base models: GPT-5.6 Luna (manager, search, retrieval); GPT-4o (vision); Whisper-1 (audio)", "score": 0.5116279069767442, "raw_score": 0.5116279069767442, "raw_scale": 1, "source_record": "Federal-KB FedQES (GPT-5.6 Luna)|GPT-5.6 Luna (manager, search, retrieval); GPT-4o (vision); Whisper-1 (audio)|2026-09-22", "source_date": "2026-09-22", "reported_by": "Federal-KB", "report_url": "https://github.com/kesfc/Federal-KB" }, { "id": "ed2265d3bcb6fa1af682", "model": "OpenHands-Versa", "organization": "CMU, All Hands AI", "configuration": "Base models: claude-sonnet-4-20250514", "score": 0.5116279069767442, "raw_score": 0.5116279069767442, "raw_scale": 1, "source_record": "OpenHands-Versa|claude-sonnet-4-20250514|2025-06-09", "source_date": "2025-06-09", "reported_by": "CMU, All Hands AI", "report_url": "https://github.com/adityasoni9998/OpenHands-Versa" }, { "id": "47ff22021f7903a46b6d", "model": "🐳 AWorld (Click to Run Instantly)", "organization": "Ant Group", "configuration": "Base models: GPT-4o, DeepSeek V3, Claude-3.7-Sonnet", "score": 0.5049833887043189, "raw_score": 0.5049833887043189, "raw_scale": 1, "source_record": "🐳 AWorld (Click to Run Instantly)|GPT-4o, DeepSeek V3, Claude-3.7-Sonnet|2025-05-23", "source_date": "2025-05-23", "reported_by": "Ant Group", "report_url": "https://github.com/inclusionAI/AWorld/tree/main" }, { "id": "a36fca6fc63e5ba82e3f", "model": "open Deep Research | pass@1", "organization": "HF 🤗 smolagents", "configuration": "Base models: o1", "score": 0.4983388704318937, "raw_score": 0.4983388704318937, "raw_scale": 1, "source_record": "open Deep Research | pass@1|o1|2025-02-10", "source_date": "2025-02-10", "reported_by": "HF 🤗 smolagents", "report_url": "https://huggingface.co/blog/open-deep-research" }, { "id": "3a33f39ed654123498b0", "model": "Codex CLI 0.149.1 Corrected", "organization": "AgentEval", "configuration": "Base models: deepseek-v4-flash-0731", "score": 0.4883720930232558, "raw_score": 0.4883720930232558, "raw_scale": 1, "source_record": "Codex CLI 0.149.1 Corrected|deepseek-v4-flash-0731|2026-09-02", "source_date": "2026-09-02", "reported_by": "AgentEval", "report_url": "https://help.aliyun.com/zh/model-studio/deepseek-v4-flash" }, { "id": "8641766722545ff4720e", "model": "h2oGPTe Agent v1.6.27 (Open Weights)", "organization": "h2o.ai", "configuration": "Base models: DeepSeek-R1 & Qwen2.5-VL-72B-Instruct", "score": 0.4883720930232558, "raw_score": 0.4883720930232558, "raw_scale": 1, "source_record": "h2oGPTe Agent v1.6.27 (Open Weights)|DeepSeek-R1 & Qwen2.5-VL-72B-Instruct|2025-04-10", "source_date": "2025-04-10", "reported_by": "h2o.ai", "report_url": "https://genai.h2o.ai/" }, { "id": "edf86d5d8c1d18c29c3b", "model": "Vector Assist", "organization": "Vector Ventures", "configuration": "Base models: Kimi-K2.5, Minimax", "score": 0.48172757475083056, "raw_score": 0.48172757475083056, "raw_scale": 1, "source_record": "Vector Assist|Kimi-K2.5, Minimax|2026-02-02", "source_date": "2026-02-02", "reported_by": "Vector Ventures", "report_url": "https://ollama.com/library/kimi-k2.5" }, { "id": "5cbd5afc19c8851343d7", "model": "Auto-Use", "organization": "Auto-Use", "configuration": "Base models: gemini-3.1-pro (openrouter)", "score": 0.4717607973421927, "raw_score": 0.4717607973421927, "raw_scale": 1, "source_record": "Auto-Use|gemini-3.1-pro (openrouter)|2026-05-21", "source_date": "2026-05-21", "reported_by": "Auto-Use", "report_url": "https://openrouter.ai/google/gemini-3.1-pro-preview" }, { "id": "80be2e15b500759b591d", "model": "TapeAgents & BrowserGym", "organization": "ServiceNow Research", "configuration": "Base models: claude-3.7-sonnet", "score": 0.4717607973421927, "raw_score": 0.4717607973421927, "raw_scale": 1, "source_record": "TapeAgents & BrowserGym|claude-3.7-sonnet|2025-03-05", "source_date": "2025-03-05", "reported_by": "ServiceNow Research", "report_url": "https://github.com/ServiceNow/TapeAgents/tree/main/examples/gaia_agent" }, { "id": "a792646af49fb5256d81", "model": "sonnet-4.6-s1-react", "organization": "Independent", "configuration": "Base models: claude-sonnet-4-5", "score": 0.4684385382059801, "raw_score": 0.4684385382059801, "raw_scale": 1, "source_record": "sonnet-4.6-s1-react|claude-sonnet-4-5|2026-07-18", "source_date": "2026-07-18", "reported_by": "Independent", "report_url": "https://arxiv.org/abs/2506.08529" }, { "id": "9f4b313774880756eb4e", "model": "rect", "organization": "person", "configuration": "Base models: qwen", "score": 0.4485049833887043, "raw_score": 0.4485049833887043, "raw_scale": 1, "source_record": "rect|qwen|2026-03-08", "source_date": "2026-03-08", "reported_by": "person", "report_url": "https://github.com/QwenLM/Qwen3" }, { "id": "2b9bafb1ae489e59c8a6", "model": "Magentic-UI", "organization": "MSR AI Frontiers", "configuration": "Base models: o4-mini", "score": 0.42524916943521596, "raw_score": 0.42524916943521596, "raw_scale": 1, "source_record": "Magentic-UI|o4-mini|2025-05-24", "source_date": "2025-05-24", "reported_by": "MSR AI Frontiers", "report_url": "https://github.com/microsoft/magentic-ui" }, { "id": "0a58d4f5a37fcafc1022", "model": "my_agent", "organization": "snikmas", "configuration": "Base models: DeepSeek-V4.1-Flash", "score": 0.4119601328903654, "raw_score": 0.4119601328903654, "raw_scale": 1, "source_record": "my_agent|DeepSeek-V4.1-Flash|2026-10-29", "source_date": "2026-10-29", "reported_by": "snikmas", "report_url": "https://huggingface.co/spaces/snikmas/my_space/tree/main" }, { "id": "a0d65ffb4c78dc880cf3", "model": "Trase Agent v0.2", "organization": "Trase Systems", "configuration": "Base models: Multi-Agent - Gemini Fine-tuned GPT-4o o1-preview", "score": 0.3953488372093023, "raw_score": 0.3953488372093023, "raw_scale": 1, "source_record": "Trase Agent v0.2|Multi-Agent - Gemini Fine-tuned GPT-4o o1-preview|2024-10-11", "source_date": "2024-10-11", "reported_by": "Trase Systems", "report_url": "https://www.trasesystems.com/" }, { "id": "bbdb07a7949807bea414", "model": "DynaSaur 🦖", "organization": "Adobe Research", "configuration": "Base models: GPT-4o (gpt-4o-2024-08-06)", "score": 0.38205980066445183, "raw_score": 0.38205980066445183, "raw_scale": 1, "source_record": "DynaSaur 🦖|GPT-4o (gpt-4o-2024-08-06)|2025-06-04", "source_date": "2025-06-04", "reported_by": "Adobe Research", "report_url": "https://github.com/adobe-research/dynasaur" }, { "id": "87b6e7ac2fc4ca34bad9", "model": "Magentic-1 (o1)", "organization": "MSR AI Frontiers", "configuration": "Base models: GPT-4o and o1-preview (varies by agent)", "score": 0.3787375415282392, "raw_score": 0.3787375415282392, "raw_scale": 1, "source_record": "Magentic-1 (o1)|GPT-4o and o1-preview (varies by agent)|2024-10-19", "source_date": "2024-10-19", "reported_by": "MSR AI Frontiers", "report_url": "https://github.com/microsoft/autogen/tree/main/python/packages/autogen-magentic-one" }, { "id": "21e9e2608b92f235059f", "model": "ADK-GAIA", "organization": "SZU-GML", "configuration": "Base models: Gemini", "score": 0.3754152823920266, "raw_score": 0.3754152823920266, "raw_scale": 1, "source_record": "ADK-GAIA|Gemini|2025-05-11", "source_date": "2025-05-11", "reported_by": "SZU-GML", "report_url": "https://github.com/GML-FMGroup/adk-gaia" }, { "id": "6be7b6021cc7aaac81e8", "model": "Trase Agent v0.1", "organization": "Trase Systems", "configuration": "Base models: Fine-tuned GPT-4o", "score": 0.3554817275747508, "raw_score": 0.3554817275747508, "raw_scale": 1, "source_record": "Trase Agent v0.1|Fine-tuned GPT-4o|2024-09-04", "source_date": "2024-09-04", "reported_by": "Trase Systems", "report_url": "https://trasesystems.com/" }, { "id": "a1c8d4779e75e8734d18", "model": "Sibyl System v0.2", "organization": "Baichuan Inc.", "configuration": "Base models: GPT-4o", "score": 0.34551495016611294, "raw_score": 0.34551495016611294, "raw_scale": 1, "source_record": "Sibyl System v0.2|GPT-4o|2023-11-03", "source_date": "2023-11-03", "reported_by": "Baichuan Inc.", "report_url": "https://sibyl.wangyulong.io/" }, { "id": "fc30d9244a457c0abe50", "model": "GenericAgent", "organization": "KW", "configuration": "Base models: Claude", "score": 0.33554817275747506, "raw_score": 0.33554817275747506, "raw_scale": 1, "source_record": "GenericAgent|Claude|2026-06-17", "source_date": "2026-06-17", "reported_by": "KW", "report_url": "https://github.com/lsdefine/GenericAgent" }, { "id": "298f7629e0c7a398da8d", "model": "AgentKB Pass1 Frozen final-301-of-301-20260826", "organization": "FedAgentKE", "configuration": "Base models: Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)", "score": 0.33222591362126247, "raw_score": 0.33222591362126247, "raw_scale": 1, "source_record": "AgentKB Pass1 Frozen final-301-of-301-20260826|Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)|2026-08-25", "source_date": "2026-08-25", "reported_by": "FedAgentKE", "report_url": "https://huggingface.co/WeihaoLi" }, { "id": "1fd3771d259bc44a5e06", "model": "Hugging Face Agents + GPT-4o", "organization": "Hugging Face 🤗", "configuration": "Base models: GPT-4o", "score": 0.33222591362126247, "raw_score": 0.33222591362126247, "raw_scale": 1, "source_record": "Hugging Face Agents + GPT-4o|GPT-4o|2024-06-27", "source_date": "2024-06-27", "reported_by": "Hugging Face 🤗", "report_url": "https://huggingface.co/learn/cookbook/agents" }, { "id": "b8ebc2a2c37df5e96313", "model": "Magentic-1", "organization": "MSR AI Frontiers", "configuration": "Base models: GPT-4o", "score": 0.3222591362126246, "raw_score": 0.3222591362126246, "raw_scale": 1, "source_record": "Magentic-1|GPT-4o|2024-10-14", "source_date": "2024-10-14", "reported_by": "MSR AI Frontiers", "report_url": "https://github.com/microsoft/autogen/tree/main/python/packages/autogen-magentic-one" }, { "id": "801acf454983d9702828", "model": "Multi-Agent Experiment v0.1 (powered by AutoGen)", "organization": "MSR AI Frontiers", "configuration": "Base models: GPT-4-turbo", "score": 0.3222591362126246, "raw_score": 0.3222591362126246, "raw_scale": 1, "source_record": "Multi-Agent Experiment v0.1 (powered by AutoGen)|GPT-4-turbo|2024-03-01", "source_date": "2024-03-01", "reported_by": "MSR AI Frontiers", "report_url": "https://aka.ms/gaia_multiagent_v01_march_1st" }, { "id": "92e1ce59d6cbb0887342", "model": "AgentKB Pass1 Frozen final-301-of-301", "organization": "FedAgentKE", "configuration": "Base models: Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)", "score": 0.2956810631229236, "raw_score": 0.2956810631229236, "raw_scale": 1, "source_record": "AgentKB Pass1 Frozen final-301-of-301|Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)|2026-08-24", "source_date": "2026-08-24", "reported_by": "FedAgentKE", "report_url": "https://huggingface.co/WeihaoLi" }, { "id": "7f6509ddea03ef50c0db", "model": "FedQES L1 rerun 20260929 - original L2 L3", "organization": "FedAgentKE", "configuration": "Base models: Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)", "score": 0.2956810631229236, "raw_score": 0.2956810631229236, "raw_scale": 1, "source_record": "FedQES L1 rerun 20260929 - original L2 L3|Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)|2026-09-30", "source_date": "2026-09-30", "reported_by": "FedAgentKE", "report_url": "https://github.com/kesfc/Federal-KB" }, { "id": "513d9533b7a3e890f79c", "model": "AgentKB FedQES Pass1 final-301-of-301", "organization": "FedAgentKE", "configuration": "Base models: Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)", "score": 0.27906976744186046, "raw_score": 0.27906976744186046, "raw_scale": 1, "source_record": "AgentKB FedQES Pass1 final-301-of-301|Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)|2026-08-28", "source_date": "2026-08-28", "reported_by": "FedAgentKE", "report_url": "https://huggingface.co/WeihaoLi" }, { "id": "6f94602f2d5f89727285", "model": "FedQES-Test-301-Final-20260903", "organization": "FedAgentKE", "configuration": "Base models: Qwen3-Next-80B-A3B-Instruct (Vertex AI MaaS)", "score": 0.26245847176079734, "raw_score": 0.26245847176079734, "raw_scale": 1, "source_record": "FedQES-Test-301-Final-20260903|Qwen3-Next-80B-A3B-Instruct (Vertex AI MaaS)|2026-09-03", "source_date": "2026-09-03", "reported_by": "FedAgentKE", "report_url": "https://github.com/kesfc/Federal-KB" }, { "id": "3ce641a024f14c79cb09", "model": "DR_agent", "organization": "Independent researcher", "configuration": "Base models: DeepSeek-V3.2", "score": 0.2558139534883721, "raw_score": 0.2558139534883721, "raw_scale": 1, "source_record": "DR_agent|DeepSeek-V3.2|2026-04-04", "source_date": "2026-04-04", "reported_by": "Independent researcher", "report_url": "https://github.com/jacksong-sourse/DR_agent" }, { "id": "e39623818242e9367903", "model": "MAYA_v1", "organization": "Adya", "configuration": "Base models: Claude-sonnet-4.5", "score": 0.25249169435215946, "raw_score": 0.25249169435215946, "raw_scale": 1, "source_record": "MAYA_v1|Claude-sonnet-4.5|2025-11-05", "source_date": "2025-11-05", "reported_by": "Adya", "report_url": "https://adya.ai/maya" }, { "id": "be3ee538cdfdf83f5303", "model": "UK AI Safety Institute Internal", "organization": "UK AI Safety Institute", "configuration": "Base models: GPT-4-Turbo", "score": 0.24916943521594684, "raw_score": 0.24916943521594684, "raw_scale": 1, "source_record": "UK AI Safety Institute Internal|GPT-4-Turbo|2024-04-21", "source_date": "2024-04-21", "reported_by": "UK AI Safety Institute", "report_url": "https://www.gov.uk/government/publications/ai-safety-institute-overview/introducing-the-ai-safety-institute" }, { "id": "f234227c964d7178533c", "model": "FRIDAY", "organization": "OS-Copilot", "configuration": "Base models: GPT-4-turbo", "score": 0.2425249169435216, "raw_score": 0.2425249169435216, "raw_scale": 1, "source_record": "FRIDAY|GPT-4-turbo|2024-01-24", "source_date": "2024-01-24", "reported_by": "OS-Copilot", "report_url": "https://os-copilot.github.io/" }, { "id": "573c0d0a154a4eb7902f", "model": "deepseek-v4-flash", "organization": "chery-ai", "configuration": "Base models: deepseek-chat", "score": 0.23588039867109634, "raw_score": 0.23588039867109634, "raw_scale": 1, "source_record": "deepseek-v4-flash|deepseek-chat|2026-07-03", "source_date": "2026-07-03", "reported_by": "chery-ai", "report_url": "https://api.deepseek.com/v1" }, { "id": "1a92dcf07887102f3145", "model": "haiku-4.5-s1-react", "organization": "Independent", "configuration": "Base models: claude-haiku-4-5", "score": 0.23588039867109634, "raw_score": 0.23588039867109634, "raw_scale": 1, "source_record": "haiku-4.5-s1-react|claude-haiku-4-5|2026-07-17", "source_date": "2026-07-17", "reported_by": "Independent", "report_url": "https://arxiv.org/abs/2506.08529" }, { "id": "d428747dc90d20dab71d", "model": "GLM-5.2 smolagents baseline partial-71", "organization": "1g0rp4vl", "configuration": "Base models: GLM-5.2", "score": 0.2292358803986711, "raw_score": 0.2292358803986711, "raw_scale": 1, "source_record": "GLM-5.2 smolagents baseline partial-71|GLM-5.2|2026-08-25", "source_date": "2026-08-25", "reported_by": "1g0rp4vl", "report_url": "https://huggingface.co/zai-org/GLM-5.2" }, { "id": "3352de8f859e36518f38", "model": "FRIDAY_without_learning", "organization": "OS-Copilot", "configuration": "Base models: GPT-4-turbo", "score": 0.2159468438538206, "raw_score": 0.2159468438538206, "raw_scale": 1, "source_record": "FRIDAY_without_learning|GPT-4-turbo|2024-01-30", "source_date": "2024-01-30", "reported_by": "OS-Copilot", "report_url": "https://os-copilot.github.io/" }, { "id": "5907cf8f05ddf63df474", "model": "FedAgentKE ExpGraph Test current 238-of-301", "organization": "FedAgentKE", "configuration": "Base models: Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)", "score": 0.19933554817275748, "raw_score": 0.19933554817275748, "raw_scale": 1, "source_record": "FedAgentKE ExpGraph Test current 238-of-301|Qwen3-Next-80B-A3B-Instruct (Vertex MaaS)|2026-08-30", "source_date": "2026-08-30", "reported_by": "FedAgentKE", "report_url": "https://huggingface.co/WeihaoLi" }, { "id": "cfc32d6e28dc79ed77a9", "model": "Hybrid-Search", "organization": "Institute of Automation Chinese Academy of Sciences", "configuration": "Base models: Qwen3.6-35B-A3B", "score": 0.16943521594684385, "raw_score": 0.16943521594684385, "raw_scale": 1, "source_record": "Hybrid-Search|Qwen3.6-35B-A3B|2026-06-30", "source_date": "2026-06-30", "reported_by": "Institute of Automation Chinese Academy of Sciences", "report_url": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B" }, { "id": "a2f240796da9f6459d71", "model": "Prometheus17", "organization": "2AcreStudios", "configuration": "Base models: Claude Opus 4.6 + Cascading Router", "score": 0.16943521594684385, "raw_score": 0.16943521594684385, "raw_scale": 1, "source_record": "Prometheus17|Claude Opus 4.6 + Cascading Router|2026-02-18", "source_date": "2026-02-18", "reported_by": "2AcreStudios", "report_url": "https://github.com/2AcreStudios/prometheus" }, { "id": "2896192ee3ff0ab889de", "model": "MAYA", "organization": "Adya", "configuration": "Base models: Claude 4.5 Sonnet", "score": 0.15946843853820597, "raw_score": 0.15946843853820597, "raw_scale": 1, "source_record": "MAYA|Claude 4.5 Sonnet|2025-11-03", "source_date": "2025-11-03", "reported_by": "Adya", "report_url": "https://adya.ai/maya" }, { "id": "385f5c93c23e708c7426", "model": "MyGAIA_Agent_v1", "organization": "Amal Alexander", "configuration": "Base models: smolagents + Mistral Small", "score": 0.15614617940199335, "raw_score": 0.15614617940199335, "raw_scale": 1, "source_record": "MyGAIA_Agent_v1|smolagents + Mistral Small|2026-04-16", "source_date": "2026-04-16", "reported_by": "Amal Alexander", "report_url": "https://huggingface.co/amallx1997" }, { "id": "813561f41549d879d5c1", "model": "WelfareWise", "organization": "SimplifyNext", "configuration": "Base models: gpt-4.1", "score": 0.14285714285714285, "raw_score": 0.14285714285714285, "raw_scale": 1, "source_record": "WelfareWise|gpt-4.1|2025-08-15", "source_date": "2025-08-15", "reported_by": "SimplifyNext", "report_url": "https://openai.com/index/gpt-4-1/" }, { "id": "526e1c775ac26d67a245", "model": "GAIA-LLM-Agent (Llama-3-70B)", "organization": "Northeastern University", "configuration": "Base models: Meta-Llama-3-70B-Instruct-Turbo", "score": 0.13953488372093023, "raw_score": 0.13953488372093023, "raw_scale": 1, "source_record": "GAIA-LLM-Agent (Llama-3-70B)|Meta-Llama-3-70B-Instruct-Turbo|2025-06-03", "source_date": "2025-06-03", "reported_by": "Northeastern University", "report_url": "https://together.ai/models/meta-llama/Meta-Llama-3-70B-Instruct-Turbo" }, { "id": "9052f828d0e34343f7ab", "model": "FedAgentKE ExpGraph Global-only 151-of-301 20260901", "organization": "FedAgentKE", "configuration": "Base models: Qwen3-Next-80B-A3B-Instruct", "score": 0.12956810631229235, "raw_score": 0.12956810631229235, "raw_scale": 1, "source_record": "FedAgentKE ExpGraph Global-only 151-of-301 20260901|Qwen3-Next-80B-A3B-Instruct|2026-08-31", "source_date": "2026-08-31", "reported_by": "FedAgentKE", "report_url": "https://github.com/kesfc/Federal-KB" }, { "id": "f01e4d15305ff7049b26", "model": "deepseek", "organization": "CG05", "configuration": "Base models: ollama", "score": 0.12956810631229235, "raw_score": 0.12956810631229235, "raw_scale": 1, "source_record": "deepseek|ollama|2026-02-11", "source_date": "2026-02-11", "reported_by": "CG05", "report_url": "https://www.deepseek.com/" }, { "id": "05a02565d125ef9f8bf2", "model": "AOne超级连接", "organization": "天翼云科技有限公司", "configuration": "Base models: DeepSeek、Qwen3", "score": 0.12292358803986711, "raw_score": 0.12292358803986711, "raw_scale": 1, "source_record": "AOne超级连接|DeepSeek、Qwen3|2025-08-07", "source_date": "2025-08-07", "reported_by": "天翼云科技有限公司", "report_url": "https://www.ctyun.cn/products/accessone" }, { "id": "ff2ff232f859345a4407", "model": "ACE-Qwen3-32B-AWQ-NoSearch", "organization": "中科大", "configuration": "Base models: Qwen3", "score": 0.11627906976744186, "raw_score": 0.11627906976744186, "raw_scale": 1, "source_record": "ACE-Qwen3-32B-AWQ-NoSearch|Qwen3|2026-08-21", "source_date": "2026-08-21", "reported_by": "中科大", "report_url": "https://huggingface.co/Qwen/Qwen3-32B-AWQ" }, { "id": "b822ab386e72df42a953", "model": "Codex web-assisted Level3 diagnostic", "organization": "Independent", "configuration": "Base models: OpenAI Codex (GPT-5 family)", "score": 0.11627906976744186, "raw_score": 0.11627906976744186, "raw_scale": 1, "source_record": "Codex web-assisted Level3 diagnostic|OpenAI Codex (GPT-5 family)|2026-08-09", "source_date": "2026-08-09", "reported_by": "Independent", "report_url": "https://openai.com/codex/" }, { "id": "7191cc684655ddeb9774", "model": "gemma-4-26b-a4b-it", "organization": "Personal", "configuration": "Base models: Google:Gemma", "score": 0.08305647840531562, "raw_score": 0.08305647840531562, "raw_scale": 1, "source_record": "gemma-4-26b-a4b-it\n|Google:Gemma\n|2026-05-12", "source_date": "2026-05-12", "reported_by": "Personal", "report_url": "https://openrouter.ai/google/gemma-4-26b-a4b-it:free" }, { "id": "8249d0961960037ef705", "model": "Prometheus", "organization": "2AcreStudios", "configuration": "Base models: Claude Opus 4.5 + Cascading Router", "score": 0.07973421926910298, "raw_score": 0.07973421926910298, "raw_scale": 1, "source_record": "Prometheus|Claude Opus 4.5 + Cascading Router|2026-01-19", "source_date": "2026-01-19", "reported_by": "2AcreStudios", "report_url": "https://huggingface.co/2AcreStudios" }, { "id": "24fda380003ad9107972", "model": "Prometheus2", "organization": "2AcreStudios", "configuration": "Base models: Claude Opus 4.5 + Cascading Router", "score": 0.07973421926910298, "raw_score": 0.07973421926910298, "raw_scale": 1, "source_record": "Prometheus2|Claude Opus 4.5 + Cascading Router|2026-01-21", "source_date": "2026-01-21", "reported_by": "2AcreStudios", "report_url": "https://github.com/2AcreStudios" }, { "id": "3ae4b3a28bdd98fe5852", "model": "GPT4 Turbo", "organization": "GAIA authors", "configuration": "Base models: GPT4", "score": 0.0664451827242525, "raw_score": 0.0664451827242525, "raw_scale": 1, "source_record": "GPT4 Turbo |GPT4|2023-11-14", "source_date": "2023-11-14", "reported_by": "GAIA authors", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "6c1da2b41182fac01710", "model": "Prometheus (Claude + Cascading Router)", "organization": "2AcreStudios", "configuration": "Base models: Claude claude-opus-4-5-20251101 + Qwen/QwQ-32B + Groq", "score": 0.0664451827242525, "raw_score": 0.0664451827242525, "raw_scale": 1, "source_record": "Prometheus (Claude + Cascading Router)|Claude claude-opus-4-5-20251101 + Qwen/QwQ-32B + Groq|2026-01-11", "source_date": "2026-01-11", "reported_by": "2AcreStudios", "report_url": "https://anthropic.com/claude" }, { "id": "2c1324d69300201936b7", "model": "personal", "organization": "person", "configuration": "Base models: qwen", "score": 0.053156146179401995, "raw_score": 0.053156146179401995, "raw_scale": 1, "source_record": "personal|qwen|2026-04-28", "source_date": "2026-04-28", "reported_by": "person", "report_url": "https://github.com/QwenLM/Qwen3" }, { "id": "dad1bb51674532d23c34", "model": "AutoGPT4", "organization": "AutoGPT", "configuration": "Base models: GPT4 + AutoGPT", "score": 0.04983388704318937, "raw_score": 0.04983388704318937, "raw_scale": 1, "source_record": "AutoGPT4|GPT4 + AutoGPT|2023-11-03", "source_date": "2023-11-03", "reported_by": "AutoGPT", "report_url": "https://github.com/Significant-Gravitas/AutoGPT" }, { "id": "dd1de84b99c11c06ca72", "model": "Shenova AI", "organization": "Neha Bhat", "configuration": "Base models: Qwen/Qwen2.5-1.5B-Instruct", "score": 0.046511627906976744, "raw_score": 0.046511627906976744, "raw_scale": 1, "source_record": "Shenova AI|Qwen/Qwen2.5-1.5B-Instruct|2026-08-23", "source_date": "2026-08-23", "reported_by": "Neha Bhat", "report_url": "https://huggingface.co/spaces/nehabhat345/shenova-ai" }, { "id": "bbf3669aa2244031bee6", "model": "GPT4", "organization": "GAIA authors", "configuration": "Base models: GPT4", "score": 0.03986710963455149, "raw_score": 0.03986710963455149, "raw_scale": 1, "source_record": "GPT4|GPT4|2023-11-09", "source_date": "2023-11-09", "reported_by": "GAIA authors", "report_url": "https://arxiv.org/abs/2303.08774" }, { "id": "9a33cfa9766d59b7a2a6", "model": "Codex CLI 0.149.1", "organization": "AgentEval", "configuration": "Base models: deepseek-v4-flash-0731", "score": 0.036544850498338874, "raw_score": 0.036544850498338874, "raw_scale": 1, "source_record": "Codex CLI 0.149.1|deepseek-v4-flash-0731|2026-09-02", "source_date": "2026-09-02", "reported_by": "AgentEval", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731" }, { "id": "b78a3be10daf5b1e6a91", "model": "GAIA-", "organization": "Self", "configuration": "Base models: deepseek-r1-distill-qwen-7b", "score": 0.036544850498338874, "raw_score": 0.036544850498338874, "raw_scale": 1, "source_record": "GAIA-|deepseek-r1-distill-qwen-7b|2025-05-31", "source_date": "2025-05-31", "reported_by": "Self", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" }, { "id": "268d7c28c6b1291541e6", "model": "GPT3.5", "organization": "GAIA authors", "configuration": "Base models: GPT3", "score": 0.026578073089700997, "raw_score": 0.026578073089700997, "raw_scale": 1, "source_record": "GPT3.5|GPT3|2023-11-17", "source_date": "2023-11-17", "reported_by": "GAIA authors", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "0ee47890924015617e7a", "model": "Custom-ReAct-Agent-v1", "organization": "Independent", "configuration": "Base models: GPT-4o", "score": 0.016611295681063124, "raw_score": 0.016611295681063124, "raw_scale": 1, "source_record": "Custom-ReAct-Agent-v1|GPT-4o|2026-09-14", "source_date": "2026-09-14", "reported_by": "Independent", "report_url": "https://github.com/tu-usuario/tu-agente](https://github.com/tu-usuario/tu-agente" }, { "id": "a9ad55c08f4cee43edf7", "model": "AG-GAIA-1", "organization": "AG", "configuration": "Base models: Claude 102", "score": 0.006644518272425249, "raw_score": 0.006644518272425249, "raw_scale": 1, "source_record": "AG-GAIA-1|Claude 102|2026-04-20", "source_date": "2026-04-20", "reported_by": "AG", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "1dfd3fb3db01ba3d5be2", "model": "gaia_agent_huggingfacet", "organization": "Individual", "configuration": "Base models: llama-3.1", "score": 0.006644518272425249, "raw_score": 0.006644518272425249, "raw_scale": 1, "source_record": "gaia_agent_huggingfacet|llama-3.1|2026-03-08", "source_date": "2026-03-08", "reported_by": "Individual", "report_url": "https://github.com/Ppp111ppp111/gaia_agent_huggingface" }, { "id": "d14cfaa38194ce649120", "model": "test-101", "organization": "TIE", "configuration": "Base models: Claude", "score": 0.006644518272425249, "raw_score": 0.006644518272425249, "raw_scale": 1, "source_record": "test-101|Claude|2026-05-13", "source_date": "2026-05-13", "reported_by": "TIE", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "85cc5888623336191ea1", "model": "AG-GAIA", "organization": "AG", "configuration": "Base models: Claude", "score": 0.0033222591362126247, "raw_score": 0.0033222591362126247, "raw_scale": 1, "source_record": "AG-GAIA|Claude|2026-04-19", "source_date": "2026-04-19", "reported_by": "AG", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "645c51132f63ad5b4cc9", "model": "AG-GAIA-3", "organization": "AG", "configuration": "Base models: Claude", "score": 0.0033222591362126247, "raw_score": 0.0033222591362126247, "raw_scale": 1, "source_record": "AG-GAIA-3|Claude|2026-04-23", "source_date": "2026-04-23", "reported_by": "AG", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "ca83cee75ecb3d6680fc", "model": "AG-GAIA-5", "organization": "AG", "configuration": "Base models: Claude", "score": 0.0033222591362126247, "raw_score": 0.0033222591362126247, "raw_scale": 1, "source_record": "AG-GAIA-5|Claude|2026-04-28", "source_date": "2026-04-28", "reported_by": "AG", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "63e9572bb19acd8fe97d", "model": "AG-GAIA-7", "organization": "AG", "configuration": "Base models: Claude", "score": 0.0033222591362126247, "raw_score": 0.0033222591362126247, "raw_scale": 1, "source_record": "AG-GAIA-7|Claude|2026-05-04", "source_date": "2026-05-04", "reported_by": "AG", "report_url": "https://huggingface.co/Arnav3035/garuda-sql-2b" }, { "id": "de9aafa01a3e57ce06de", "model": "Flywork CLI 0.2.4-beta.1", "organization": "AgentEval", "configuration": "Base models: Qwen3.8-27B-NVFP4", "score": 0.0033222591362126247, "raw_score": 0.0033222591362126247, "raw_scale": 1, "source_record": "Flywork CLI 0.2.4-beta.1|Qwen3.8-27B-NVFP4|2026-09-01", "source_date": "2026-09-01", "reported_by": "AgentEval", "report_url": "https://qwen.ai/blog?id=qwen3.8" }, { "id": "b7eda3b04eb7dad307b5", "model": "GAIA Agent (Gemini 2.0 Flash)", "organization": "Karukola Hemanth", "configuration": "Base models: Gemini 2.0 Flash", "score": 0.0033222591362126247, "raw_score": 0.0033222591362126247, "raw_scale": 1, "source_record": "GAIA Agent (Gemini 2.0 Flash)\n|Gemini 2.0 Flash\n|2026-10-04", "source_date": "2026-10-04", "reported_by": "Karukola Hemanth", "report_url": "https://github.com/Karukolahemanth/Hugging-face" } ] }, { "id": "swe-bench-pro-v2-full", "name": "SWE-bench Pro V2 (Full)", "group": "编程", "unit": "%", "metric": "Resolved", "protocol": { "en": "Scale V2 Full, 642 tasks. Model and agent settings follow the published entries.", "zh": "Scale V2 Full，642 道任务。模型与 Agent 配置沿用发布名称。" }, "source": { "data_url": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "sha256": "b8efacea23d2a9b326009f31d08043d78e5c55a17999f94a9ce354028b9ce819", "retrieved_at": "2026-11-02T05:57:36.682977+00:00", "name": "Scale AI", "url": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "selector": "Next page variants[key=full].entries; exclude deprecated; score / 100", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "57a380a901f32a4b67b0", "model": "Opus 5 (Claude Code) xhigh", "organization": "anthropic", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9940000000000001, "raw_score": 99.4, "raw_scale": 100, "source_record": "Opus 5 (Claude Code) xhigh|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 0.4 }, { "id": "5d626585feb5cdba3837", "model": "Gemini 4 Argon (mini-swe-agent) high", "organization": "openai", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9922, "raw_score": 99.22, "raw_scale": 100, "source_record": "Gemini 4 Argon (mini-swe-agent) high|V2 Full; harness as listed in the model name", "source_date": "2026-10-14T16:34:00.000Z", "confidence_interval": 0.45 }, { "id": "975126e1b8c30068f6c2", "model": "Fable 5.1 (Claude Code) high", "organization": "anthropic", "configuration": "V2 Full; harness as listed in the model name", "score": 0.991, "raw_score": 99.1, "raw_scale": 100, "source_record": "Fable 5.1 (Claude Code) high|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 0.5 }, { "id": "e00a85476d4c9aebd8fe", "model": "Kimi-K3 (mini-swe-agent) max", "organization": "moonshot", "configuration": "V2 Full; harness as listed in the model name", "score": 0.977, "raw_score": 97.7, "raw_scale": 100, "source_record": "Kimi-K3 (mini-swe-agent) max|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 0.9 }, { "id": "0e60bfa3bc98b58ed806", "model": "GPT-6-Astra (Codex) high", "organization": "openai", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9690000000000001, "raw_score": 96.9, "raw_scale": 100, "source_record": "GPT-6-Astra (Codex) high|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 1.1 }, { "id": "cd50026a8a790d380694", "model": "Muse Wonder 1.0 (mini-swe-agent) xhigh", "organization": "openai", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9673, "raw_score": 96.73, "raw_scale": 100, "source_record": "Muse Wonder 1.0 (mini-swe-agent) xhigh|V2 Full; harness as listed in the model name", "source_date": "2026-10-14T17:57:23.000Z", "confidence_interval": 1.12 }, { "id": "18a458395905490e53db", "model": "GPT-6.1-Sol (mini-swe-agent) xhigh", "organization": "openai", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9611, "raw_score": 96.11, "raw_scale": 100, "source_record": "GPT-6.1-Sol (mini-swe-agent) xhigh|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 1.24 }, { "id": "350af95f736554c87eb7", "model": "Sonnet 5.5 (mini-swe-agent) xhigh", "organization": "anthropic", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9579000000000001, "raw_score": 95.79, "raw_scale": 100, "source_record": "Sonnet 5.5 (mini-swe-agent) xhigh|V2 Full; harness as listed in the model name", "source_date": "2026-10-12T23:21:32.000Z", "confidence_interval": 1.3 }, { "id": "4b792328587d9af94e9f", "model": "GLM-5.3 (mini-swe-agent) max", "organization": "zai", "configuration": "V2 Full; harness as listed in the model name", "score": 0.956, "raw_score": 95.6, "raw_scale": 100, "source_record": "GLM-5.3 (mini-swe-agent) max|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 1.3 }, { "id": "2b06301af8a0f2dff10d", "model": "GPT-5.6-sol (Codex) xhigh", "organization": "openai", "configuration": "V2 Full; harness as listed in the model name", "score": 0.955, "raw_score": 95.5, "raw_scale": 100, "source_record": "GPT-5.6-sol (Codex) xhigh|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 1.4 }, { "id": "5579b6885e78e7c8752c", "model": "Muse Spark Open (mini-swe-agent) xhigh", "organization": "openai", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9501999999999999, "raw_score": 95.02, "raw_scale": 100, "source_record": "Muse Spark Open (mini-swe-agent) xhigh|V2 Full; harness as listed in the model name", "source_date": "2026-10-16T19:17:34.000Z", "confidence_interval": 1.43 }, { "id": "2c9bdc98789cbed726c3", "model": "Gemini 3.8 Flash (mini-swe-agent) high", "organization": "google", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9486, "raw_score": 94.86, "raw_scale": 100, "source_record": "Gemini 3.8 Flash (mini-swe-agent) high|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 1.46 }, { "id": "948121250521b04912f8", "model": "Sonnet 5 (Claude Code) xhigh", "organization": "anthropic", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9315000000000001, "raw_score": 93.15, "raw_scale": 100, "source_record": "Sonnet 5 (Claude Code) xhigh|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 1.71 }, { "id": "cc2d5ebfeb4381f6b4c1", "model": "GPT-5.6-Terra (Codex) xhigh", "organization": "openai", "configuration": "V2 Full; harness as listed in the model name", "score": 0.9237000000000001, "raw_score": 92.37, "raw_scale": 100, "source_record": "GPT-5.6-Terra (Codex) xhigh|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 1.81 }, { "id": "eb8cbbc373129465b16e", "model": "Inkling (mini-swe-agent) xhigh", "organization": "thinkingmachines", "configuration": "V2 Full; harness as listed in the model name", "score": 0.8987999999999999, "raw_score": 89.88, "raw_scale": 100, "source_record": "Inkling (mini-swe-agent) xhigh|V2 Full; harness as listed in the model name", "source_date": "2026-10-07T01:44:58.000Z", "confidence_interval": 2.1 }, { "id": "e344c58c81b2eaee884e", "model": "Haiku 5.5 (mini-swe-agent) high", "organization": "anthropic", "configuration": "V2 Full; harness as listed in the model name", "score": 0.8723000000000001, "raw_score": 87.23, "raw_scale": 100, "source_record": "Haiku 5.5 (mini-swe-agent) high|V2 Full; harness as listed in the model name", "source_date": "2026-10-08T20:32:00.000Z", "confidence_interval": 2.36 } ] }, { "id": "aider-polyglot", "name": "Aider Polyglot", "group": "编程", "unit": "%", "metric": "Correct after repair", "protocol": { "en": "225 exercises across six languages. Percent correct includes a second repair attempt. Edit format and reasoning effort are retained.", "zh": "6 种语言、225 道任务；答对率包含第二次修复机会。保留编辑格式与推理强度。" }, "source": { "data_url": "https://aider.chat/docs/leaderboards/", "sha256": "cea6cab3ac65e4d41cce072fcac5540497d757b3a6f72d997b7c3607b44c470e", "retrieved_at": "2026-11-01T16:38:03.246059+00:00", "name": "Aider", "url": "https://aider.chat/docs/leaderboards/", "selector": "225 test cases; Pass rate 2; latest model/configuration, not best run", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "c552cfecc2591f3c69cd", "model": "gpt-5 (high)", "organization": "OpenAI", "configuration": "Edit format: diff; Reasoning effort: high; Versions: 0.86.2.dev", "score": 0.88, "raw_score": 88.0, "raw_scale": 100, "source_record": "2025-08-23-15-47-21--gpt-5-high", "source_date": "2025-08-23", "first_pass_score": 52.0 }, { "id": "4b114066b7dde21f930a", "model": "gpt-5 (medium)", "organization": "OpenAI", "configuration": "Edit format: diff; Reasoning effort: medium; Versions: 0.86.2.dev", "score": 0.867, "raw_score": 86.7, "raw_scale": 100, "source_record": "2025-08-25-13-23-27--gpt-5-medium", "source_date": "2025-08-25", "first_pass_score": 49.8 }, { "id": "4ef83cbfed7c85d690f6", "model": "o3-pro (high)", "organization": "OpenAI", "configuration": "Edit format: diff; Reasoning effort: high; Versions: 0.85.1.dev", "score": 0.8490000000000001, "raw_score": 84.9, "raw_scale": 100, "source_record": "2025-06-28-00-38-18--o3-pro-high", "source_date": "2025-06-28", "first_pass_score": 43.6 }, { "id": "006937df70992ab02751", "model": "gemini-2.5-pro-preview-06-05 (32k think)", "organization": "Google", "configuration": "Edit format: diff-fenced; Versions: 0.84.1.dev", "score": 0.831, "raw_score": 83.1, "raw_scale": 100, "source_record": "2025-06-06-16-36-21--gemini0605-32k-think-diff-fenced", "source_date": "2025-06-06", "first_pass_score": 46.2 }, { "id": "102fb983907532c40d46", "model": "gpt-5 (low)", "organization": "OpenAI", "configuration": "Edit format: diff; Reasoning effort: low; Versions: 0.86.2.dev", "score": 0.813, "raw_score": 81.3, "raw_scale": 100, "source_record": "2025-08-25-14-16-37--gpt-5-low", "source_date": "2025-08-25", "first_pass_score": 43.1 }, { "id": "6ab3d1d55a32f513c666", "model": "o3 (high)", "organization": "", "configuration": "Edit format: diff; Reasoning effort: high; Versions: 0.84.1.dev", "score": 0.813, "raw_score": 81.3, "raw_scale": 100, "source_record": "2025-06-25-21-04-24--o3-price-reduction-high", "source_date": "2025-06-25", "first_pass_score": 40.0 }, { "id": "345dd1ba0c1dc6dd5f94", "model": "grok-4 (high)", "organization": "xAI", "configuration": "Edit format: diff; Reasoning effort: high; Versions: 0.85.2.dev", "score": 0.7959999999999999, "raw_score": 79.6, "raw_scale": 100, "source_record": "2025-07-11-19-37-40--xai-or-grok4-high", "source_date": "2025-07-11", "first_pass_score": 40.9 }, { "id": "408979decde7db9ee58a", "model": "gemini-2.5-pro-preview-06-05 (default think)", "organization": "Google", "configuration": "Edit format: diff-fenced; Versions: 0.84.1.dev", "score": 0.7909999999999999, "raw_score": 79.1, "raw_scale": 100, "source_record": "2025-06-06-18-38-56--gemini0605-diff-fenced", "source_date": "2025-06-06", "first_pass_score": 44.9 }, { "id": "4f670a1faa0d032d2886", "model": "Gemini 2.5 Pro Preview 05-06", "organization": "Google", "configuration": "Edit format: diff-fenced; Versions: 0.82.4.dev", "score": 0.769, "raw_score": 76.9, "raw_scale": 100, "source_record": "2025-05-07-19-32-40--gemini0506-diff-fenced-completion_cost", "source_date": "2025-05-07", "first_pass_score": 36.4 }, { "id": "b4c12b58bb7fc91144c3", "model": "o3", "organization": "OpenAI", "configuration": "Edit format: diff; Versions: 0.84.1.dev", "score": 0.769, "raw_score": 76.9, "raw_scale": 100, "source_record": "2025-06-25-20-30-16--o3-price-reduction", "source_date": "2025-06-25", "first_pass_score": 40.9 }, { "id": "9d623b6f1cd749c8c5b4", "model": "DeepSeek-V3.2-Exp (Reasoner)", "organization": "DeepSeek", "configuration": "Edit format: diff; Versions: 0.86.2.dev", "score": 0.742, "raw_score": 74.2, "raw_scale": 100, "source_record": "2025-10-03-09-45-34--deepseek-v3.2-reasoner", "source_date": "2025-10-03", "first_pass_score": 39.6 }, { "id": "1e877385d232ffac6bb3", "model": "Gemini 2.5 Pro Preview 03-25", "organization": "Google", "configuration": "Edit format: diff-fenced; Versions: 0.81.3.dev", "score": 0.7290000000000001, "raw_score": 72.9, "raw_scale": 100, "source_record": "2025-04-12-04-55-50--gemini-25-pro-diff-fenced", "source_date": "2025-04-12", "first_pass_score": 40.9 }, { "id": "912aaf737dc5c1df5098", "model": "claude-opus-4-20250514 (32k thinking)", "organization": "Anthropic", "configuration": "Edit format: diff; Versions: 0.83.3.dev", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "2025-05-25-20-40-51--opus4-diff-exuser", "source_date": "2025-05-25", "first_pass_score": 37.3 }, { "id": "77ad69a4e98121e13d36", "model": "o4-mini (high)", "organization": "OpenAI", "configuration": "Edit format: diff; Versions: 0.82.1.dev", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "2025-04-16-22-01-58--o4-mini-high-diff-exsys", "source_date": "2025-04-16", "first_pass_score": 19.6 }, { "id": "65b8a066ba06551aaa09", "model": "claude-opus-4-20250514 (no think)", "organization": "Anthropic", "configuration": "Edit format: diff; Versions: 0.83.3.dev", "score": 0.7070000000000001, "raw_score": 70.7, "raw_scale": 100, "source_record": "2025-05-25-19-57-20--opus4-diff-exuser", "source_date": "2025-05-25", "first_pass_score": 32.9 }, { "id": "b944ea7eee0922515873", "model": "DeepSeek-V3.2-Exp (Chat)", "organization": "DeepSeek", "configuration": "Edit format: diff; Versions: 0.86.2.dev", "score": 0.7020000000000001, "raw_score": 70.2, "raw_scale": 100, "source_record": "2025-10-03-09-21-36--deepseek-v3.2-chat", "source_date": "2025-10-03", "first_pass_score": 38.7 }, { "id": "ef7e5c1fa9722f1fbcb8", "model": "claude-3-7-sonnet-20250219 (32k thinking tokens)", "organization": "Anthropic", "configuration": "Edit format: diff; Versions: 0.75.1.dev", "score": 0.649, "raw_score": 64.9, "raw_scale": 100, "source_record": "2025-02-24-21-47-23--sonnet37-diff-think-32k-64k", "source_date": "2025-02-24", "first_pass_score": 29.3 }, { "id": "b1e891592f06aceac6e5", "model": "DeepSeek R1 + claude-3-5-sonnet-20241022", "organization": "DeepSeek", "configuration": "Edit format: architect; Versions: 0.72.3.dev", "score": 0.64, "raw_score": 64.0, "raw_scale": 100, "source_record": "2025-01-23-19-14-48--r1-architect-sonnet", "source_date": "2025-01-23", "first_pass_score": 27.1 }, { "id": "bfb973228267eed45192", "model": "claude-sonnet-4-20250514 (32k thinking)", "organization": "Anthropic", "configuration": "Edit format: diff; Versions: 0.83.3.dev", "score": 0.613, "raw_score": 61.3, "raw_scale": 100, "source_record": "2025-05-24-22-10-36--sonnet4-diff-exuser-think32k", "source_date": "2025-05-24", "first_pass_score": 25.8 }, { "id": "230875c1b7e2d69a5cc4", "model": "claude-3-7-sonnet-20250219 (no thinking)", "organization": "Anthropic", "configuration": "Edit format: diff; Versions: 0.74.4.dev", "score": 0.604, "raw_score": 60.4, "raw_scale": 100, "source_record": "2025-02-24-19-54-07--sonnet37-diff", "source_date": "2025-02-24", "first_pass_score": 24.4 }, { "id": "1806e05880346cfd8d3b", "model": "Qwen3 235B A22B diff, no think, Alibaba API", "organization": "Alibaba", "configuration": "Edit format: diff; Versions: 0.82.4.dev", "score": 0.596, "raw_score": 59.6, "raw_scale": 100, "source_record": "2025-05-09-17-02-02--qwen3-235b-a22b.unthink_16k_diff", "source_date": "2025-05-09", "first_pass_score": 28.9 }, { "id": "d641506d457c2b102f23", "model": "Kimi K2", "organization": "Moonshot AI", "configuration": "Edit format: diff; Versions: 0.85.3.dev", "score": 0.591, "raw_score": 59.1, "raw_scale": 100, "source_record": "2025-07-17-17-41-54--kimi-k2-diff-or-pricing", "source_date": "2025-07-17", "first_pass_score": 20.4 }, { "id": "62c360ef0886d44e62b9", "model": "DeepSeek R1", "organization": "DeepSeek", "configuration": "Edit format: diff; Versions: 0.71.2.dev", "score": 0.569, "raw_score": 56.9, "raw_scale": 100, "source_record": "2025-01-20-19-11-38--ds-turns-upd-cur-msgs-fix-with-summarizer", "source_date": "2025-01-20", "first_pass_score": 26.7 }, { "id": "4bcbd0ff1d35f6d7ed2d", "model": "claude-sonnet-4-20250514 (no thinking)", "organization": "Anthropic", "configuration": "Edit format: diff; Versions: 0.83.3.dev", "score": 0.564, "raw_score": 56.4, "raw_scale": 100, "source_record": "2025-05-24-21-17-54--sonnet4-diff-exuser", "source_date": "2025-05-24", "first_pass_score": 20.4 }, { "id": "ee76acaa7cf1c23c8e25", "model": "DeepSeek V3 (0324)", "organization": "DeepSeek", "configuration": "Edit format: diff; Versions: 0.78.1.dev", "score": 0.551, "raw_score": 55.1, "raw_scale": 100, "source_record": "2025-03-24-15-41-33--deepseek-v3-0324-polyglot-diff", "source_date": "2025-03-24", "first_pass_score": 28.0 }, { "id": "04fc642c0eccb74d8575", "model": "gemini-2.5-flash-preview-05-20 (24k think)", "organization": "Google", "configuration": "Edit format: diff; Versions: 0.83.3.dev", "score": 0.551, "raw_score": 55.1, "raw_scale": 100, "source_record": "2025-05-25-22-58-44--flash25-05-20-24k-think", "source_date": "2025-05-25", "first_pass_score": 26.2 }, { "id": "5336aa72794204b20e6f", "model": "Quasar Alpha", "organization": "", "configuration": "Edit format: diff; Versions: 0.80.5.dev", "score": 0.547, "raw_score": 54.7, "raw_scale": 100, "source_record": "2025-04-04-02-57-25--qalpha-diff-exsys", "source_date": "2025-04-04", "first_pass_score": 21.8 }, { "id": "c8f7925f2707c689198f", "model": "o3-mini (medium)", "organization": "OpenAI", "configuration": "Edit format: diff; Versions: 0.72.4.dev", "score": 0.5379999999999999, "raw_score": 53.8, "raw_scale": 100, "source_record": "2025-01-31-20-27-46--o3-mini-diff2", "source_date": "2025-01-31", "first_pass_score": 19.1 }, { "id": "4d22ee37aa8df07b2328", "model": "Grok 3 Beta", "organization": "xAI", "configuration": "Edit format: diff; Versions: 0.81.2.dev", "score": 0.5329999999999999, "raw_score": 53.3, "raw_scale": 100, "source_record": "2025-04-10-04-21-31--grok3-diff-exuser", "source_date": "2025-04-10", "first_pass_score": 22.2 }, { "id": "0b467e032225b103627b", "model": "Optimus Alpha", "organization": "", "configuration": "Edit format: diff; Versions: 0.81.2.dev", "score": 0.529, "raw_score": 52.9, "raw_scale": 100, "source_record": "2025-04-10-19-02-44--oalpha-diff-exsys", "source_date": "2025-04-10", "first_pass_score": 21.3 }, { "id": "4a52e7d43a84b5965108", "model": "gpt-4.1", "organization": "OpenAI", "configuration": "Edit format: diff; Versions: 0.81.4.dev", "score": 0.524, "raw_score": 52.4, "raw_scale": 100, "source_record": "2025-04-14-21-05-54--gpt41-diff-exuser", "source_date": "2025-04-14", "first_pass_score": 20.0 }, { "id": "2fa8416b721ccf11e330", "model": "claude-3-5-sonnet-20241022", "organization": "Anthropic", "configuration": "Edit format: diff; Versions: 0.71.2.dev", "score": 0.516, "raw_score": 51.6, "raw_scale": 100, "source_record": "2025-01-17-19-44-33--sonnet-baseline-jan-17", "source_date": "2025-01-17", "first_pass_score": 22.2 }, { "id": "429774706591af14eb2f", "model": "Grok 3 Mini Beta (high)", "organization": "xAI", "configuration": "Edit format: whole; Versions: 0.81.3.dev", "score": 0.493, "raw_score": 49.3, "raw_scale": 100, "source_record": "2025-04-10-23-59-02--xai-grok3-mini-whole-high", "source_date": "2025-04-10", "first_pass_score": 17.3 }, { "id": "5adfbc9ccde37a3ecb28", "model": "DeepSeek Chat V3 (prev)", "organization": "DeepSeek", "configuration": "Edit format: diff; Versions: 0.69.2.dev", "score": 0.484, "raw_score": 48.4, "raw_scale": 100, "source_record": "2024-12-25-13-31-51--deepseekv3preview-diff2", "source_date": "2024-12-25", "first_pass_score": 22.7 }, { "id": "25242445fcfa3283bff0", "model": "gemini-2.5-flash-preview-04-17 (default)", "organization": "Google", "configuration": "Edit format: diff; Versions: 0.82.3.dev", "score": 0.47100000000000003, "raw_score": 47.1, "raw_scale": 100, "source_record": "2025-04-20-19-54-31--flash25-diff-no-think", "source_date": "2025-04-20", "first_pass_score": 21.8 }, { "id": "a41eb9bbd37a7dd36215", "model": "chatgpt-4o-latest (2025-03-29)", "organization": "OpenAI", "configuration": "Edit format: diff; Versions: 0.79.3.dev", "score": 0.45299999999999996, "raw_score": 45.3, "raw_scale": 100, "source_record": "2025-03-29-05-24-55--chatgpt4o-mar28-diff", "source_date": "2025-03-29", "first_pass_score": 16.4 }, { "id": "e106eb45b8ed91561863", "model": "gemini-2.5-flash-preview-05-20 (no think)", "organization": "Google", "configuration": "Edit format: diff; Versions: 0.83.3.dev", "score": 0.44, "raw_score": 44.0, "raw_scale": 100, "source_record": "2025-05-26-15-56-31--flash25-05-20-24k-think", "source_date": "2025-05-26", "first_pass_score": 20.9 }, { "id": "7e9bac23a4a675d3d0d7", "model": "gpt-oss-120b (high)", "organization": "OpenAI", "configuration": "Edit format: diff; Versions: 0.85.3.dev", "score": 0.418, "raw_score": 41.8, "raw_scale": 100, "source_record": "2025-08-06-04-54-48--gpt-oss-120b-high-polyglot", "source_date": "2025-08-06", "first_pass_score": 13.8 }, { "id": "95fc7325f57850a8c757", "model": "Qwen3 32B", "organization": "Alibaba", "configuration": "Edit format: diff; Versions: 0.82.4.dev", "score": 0.4, "raw_score": 40.0, "raw_scale": 100, "source_record": "2025-05-08-03-20-24--qwen3-32b-default", "source_date": "2025-05-08", "first_pass_score": 14.2 }, { "id": "7fc8aa55680413e8795b", "model": "gemini-exp-1206", "organization": "Google", "configuration": "Edit format: whole; Versions: 0.69.2.dev", "score": 0.382, "raw_score": 38.2, "raw_scale": 100, "source_record": "2024-12-22-18-43-25--gemini-exp-1206-polyglot-whole-2", "source_date": "2024-12-22", "first_pass_score": 19.6 }, { "id": "b11f9fa01b223c6a02a1", "model": "Gemini 2.0 Pro exp-02-05", "organization": "Google", "configuration": "Edit format: whole; Versions: 0.75.2.dev", "score": 0.35600000000000004, "raw_score": 35.6, "raw_scale": 100, "source_record": "2025-02-25-20-23-07--gemini-pro", "source_date": "2025-02-25", "first_pass_score": 20.4 }, { "id": "c253f2d627cb8f5f9382", "model": "Grok 3 Mini Beta (low)", "organization": "xAI", "configuration": "Edit format: whole; Versions: 0.81.2.dev", "score": 0.34700000000000003, "raw_score": 34.7, "raw_scale": 100, "source_record": "2025-04-10-18-47-24--grok3-mini-whole-exuser", "source_date": "2025-04-10", "first_pass_score": 11.1 }, { "id": "e47909a7d93514fa1155", "model": "o1-mini-2024-09-12", "organization": "OpenAI", "configuration": "Edit format: whole; Versions: 0.69.2.dev", "score": 0.32899999999999996, "raw_score": 32.9, "raw_scale": 100, "source_record": "2024-12-22-21-26-35--polyglot-o1mini-whole", "source_date": "2024-12-22", "first_pass_score": 5.8 }, { "id": "f21d2bcd7efec4395a02", "model": "gpt-4.1-mini", "organization": "OpenAI", "configuration": "Edit format: diff; Versions: 0.81.4.dev", "score": 0.324, "raw_score": 32.4, "raw_scale": 100, "source_record": "2025-04-14-21-27-53--gpt41mini-diff", "source_date": "2025-04-14", "first_pass_score": 11.1 }, { "id": "7cf35abbe3b439b0da87", "model": "claude-3-5-haiku-20241022", "organization": "Anthropic", "configuration": "Edit format: diff; Versions: 0.69.2.dev", "score": 0.28, "raw_score": 28.0, "raw_scale": 100, "source_record": "2024-12-21-21-46-27--polyglot-haiku-diff", "source_date": "2024-12-21", "first_pass_score": 7.1 }, { "id": "83715cbcfbce04922901", "model": "QwQ-32B + Qwen 2.5 Coder Instruct", "organization": "", "configuration": "Edit format: architect; Versions: 0.75.3.dev", "score": 0.262, "raw_score": 26.2, "raw_scale": 100, "source_record": "2025-03-07-15-11-27--qwq32b-arch-temp-topp-again", "source_date": "2025-03-07", "first_pass_score": 9.8 }, { "id": "5ce37490b8f384812fc8", "model": "gpt-4o-2024-08-06", "organization": "OpenAI", "configuration": "Edit format: diff; Versions: 0.70.1.dev", "score": 0.231, "raw_score": 23.1, "raw_scale": 100, "source_record": "2024-12-30-20-44-54--gpt4o-ex-as-sys-clean-prompt", "source_date": "2024-12-30", "first_pass_score": 4.9 }, { "id": "91923b40924d9cabd6fc", "model": "gemini-2.0-flash-exp", "organization": "Google", "configuration": "Edit format: whole; Versions: 0.69.2.dev", "score": 0.222, "raw_score": 22.2, "raw_scale": 100, "source_record": "2024-12-22-20-08-13--gemini-2.0-flash-exp-polyglot-whole", "source_date": "2024-12-22", "first_pass_score": 11.6 }, { "id": "f343fc7eddebfbeecc25", "model": "qwen-max-2025-01-25", "organization": "Alibaba", "configuration": "Edit format: diff; Versions: 0.72.4.dev", "score": 0.218, "raw_score": 21.8, "raw_scale": 100, "source_record": "2025-01-28-16-00-03--qwen-max-2025-01-25-polyglot-diff", "source_date": "2025-01-28", "first_pass_score": 9.3 }, { "id": "44233b39b3a887676416", "model": "QwQ-32B", "organization": "", "configuration": "Edit format: diff; Versions: 0.75.3.dev", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "2025-03-06-17-40-24--qwq32b-diff-temp-topp-ex-sys-remind-user-for-real", "source_date": "2025-03-06", "first_pass_score": 8.0 }, { "id": "93976aa93dc075cb506e", "model": "gemini-2.0-flash-thinking-exp-01-21", "organization": "Google", "configuration": "Edit format: diff; Versions: 0.72.2.dev", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "2025-01-21-22-51-49--gemini-2.0-flash-thinking-exp-01-21-polyglot-diff", "source_date": "2025-01-21", "first_pass_score": 5.8 }, { "id": "04f476044727e6198437", "model": "gpt-4o-2024-11-20", "organization": "OpenAI", "configuration": "Edit format: diff; Versions: 0.70.1.dev", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "2024-12-30-20-57-12--gpt-4o-2024-11-20-ex-as-sys", "source_date": "2024-12-30", "first_pass_score": 4.9 }, { "id": "a7b50670d8617762b33a", "model": "DeepSeek Chat V2.5", "organization": "DeepSeek", "configuration": "Edit format: diff; Versions: 0.69.2.dev", "score": 0.17800000000000002, "raw_score": 17.8, "raw_scale": 100, "source_record": "2024-12-21-20-56-21--polyglot-deepseek-diff", "source_date": "2024-12-21", "first_pass_score": 5.3 }, { "id": "2f0d181140d945822ce3", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Edit format: whole; Versions: 0.69.2.dev", "score": 0.16399999999999998, "raw_score": 16.4, "raw_scale": 100, "source_record": "2024-12-26-00-55-20--Qwen2.5-Coder-32B-Instruct", "source_date": "2024-12-26", "first_pass_score": 4.9 }, { "id": "f2155382d88c7a567813", "model": "Llama 4 Maverick", "organization": "Meta", "configuration": "Edit format: whole; Versions: 0.81.2.dev", "score": 0.156, "raw_score": 15.6, "raw_scale": 100, "source_record": "2025-04-06-08-39-52--llama-4-maverick-17b-128e-instruct-polyglot-whole", "source_date": "2025-04-06", "first_pass_score": 4.4 }, { "id": "25ef8416a086bdf9b7a9", "model": "yi-lightning", "organization": "01.AI", "configuration": "Edit format: whole; Versions: 0.69.2.dev", "score": 0.129, "raw_score": 12.9, "raw_scale": 100, "source_record": "2024-12-23-01-11-56--yi-test", "source_date": "2024-12-23", "first_pass_score": 5.8 }, { "id": "d7b121afbaf2b789a94c", "model": "command-a-03-2025-quality", "organization": "", "configuration": "Edit format: whole; Versions: 0.77.1.dev", "score": 0.12, "raw_score": 12.0, "raw_scale": 100, "source_record": "2025-03-14-23-40-00--cmda-quality-whole2", "source_date": "2025-03-14", "first_pass_score": 2.2 }, { "id": "81a21e8ed126c2a2ccf6", "model": "Codestral 25.01", "organization": "", "configuration": "Edit format: whole; Versions: 0.71.2.dev", "score": 0.111, "raw_score": 11.1, "raw_scale": 100, "source_record": "2025-01-13-18-17-25--codestral-whole2", "source_date": "2025-01-13", "first_pass_score": 4.0 }, { "id": "10395a725f8cb9cb3823", "model": "openhands-lm-32b-v0.1", "organization": "", "configuration": "Edit format: whole; Versions: 0.82.2.dev", "score": 0.102, "raw_score": 10.2, "raw_scale": 100, "source_record": "2025-04-19-14-43-04--o4-mini-patch", "source_date": "2025-04-19", "first_pass_score": 4.0 }, { "id": "088a53596ce7918d32fd", "model": "gpt-4.1-nano", "organization": "OpenAI", "configuration": "Edit format: whole; Versions: 0.81.4.dev", "score": 0.08900000000000001, "raw_score": 8.9, "raw_scale": 100, "source_record": "2025-04-14-22-46-01--gpt41nano-diff", "source_date": "2025-04-14", "first_pass_score": 3.1 }, { "id": "965213a6d29ae14a81ac", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Edit format: diff; Versions: 0.69.2.dev", "score": 0.08, "raw_score": 8.0, "raw_scale": 100, "source_record": "2024-12-22-13-22-32--polyglot-qwen-diff", "source_date": "2024-12-22", "first_pass_score": 4.4 }, { "id": "4a0471aba501663fad22", "model": "gemma-3-27b-it", "organization": "Google", "configuration": "Edit format: whole; Versions: 0.77.1.dev", "score": 0.049, "raw_score": 4.9, "raw_scale": 100, "source_record": "2025-03-15-01-21-24--gemma3-27b-or", "source_date": "2025-03-15", "first_pass_score": 1.8 }, { "id": "7aca33f03b1afdf9fac4", "model": "gpt-4o-mini-2024-07-18", "organization": "OpenAI", "configuration": "Edit format: whole; Versions: 0.69.2.dev", "score": 0.036000000000000004, "raw_score": 3.6, "raw_scale": 100, "source_record": "2024-12-21-18-41-18--polyglot-gpt-4o-mini", "source_date": "2024-12-21", "first_pass_score": 0.9 } ] }, { "id": "bigcodebench-hard-instruct", "name": "BigCodeBench Hard (Instruct)", "group": "编程", "unit": "%", "metric": "Calibrated pass@1", "protocol": { "en": "Hard-Instruct, calibrated pass@1 with greedy decoding. Each run retains its prompt and prefilling settings.", "zh": "Hard-Instruct，采用贪心解码的校准 pass@1。各条结果保留提示与预填设置。" }, "source": { "data_url": "https://bigcode-bench.github.io/results-hard.json", "sha256": "6c3b7df370bb95e3671aa70979c53a81fc0f946e29f574e3e62b225db8024d63", "retrieved_at": "2026-11-02T05:57:36.498528+00:00", "name": "BigCodeBench", "url": "https://bigcode-bench.github.io/", "selector": "Hard: pass@1.instruct; numeric scores including zero", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "8e2b7294bf3d72cb64e2", "model": "o3-mini-2025-01-31 (temperature=1, reasoning=medium)", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.331, "raw_score": 33.1, "raw_scale": 100, "source_record": "o3-mini-2025-01-31 (temperature=1, reasoning=medium)", "source_date": "2025-01-31", "report_url": "https://openai.com/index/openai-o3-mini/" }, { "id": "4fca3357dbc6a3419617", "model": "Claude-3.7-Sonnet-20250219 (temperature=1, length=12800, reasoning=3200)", "organization": "Anthropic", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.324, "raw_score": 32.4, "raw_scale": 100, "source_record": "Claude-3.7-Sonnet-20250219 (temperature=1, length=12800, reasoning=3200)", "source_date": "2025-02-19", "report_url": "https://www.anthropic.com/news/claude-3-7-sonnet" }, { "id": "b1dad343b8e41526c0f5", "model": "o1-2024-12-17 (temperature=1, reasoning=high)", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.324, "raw_score": 32.4, "raw_scale": 100, "source_record": "o1-2024-12-17 (temperature=1, reasoning=high)", "source_date": "2024-12-17", "report_url": "https://openai.com/o1/" }, { "id": "67b90159471f802a0870", "model": "o3-mini-2025-01-31 (temperature=1, reasoning=high)", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.324, "raw_score": 32.4, "raw_scale": 100, "source_record": "o3-mini-2025-01-31 (temperature=1, reasoning=high)", "source_date": "2025-01-31", "report_url": "https://openai.com/index/openai-o3-mini/" }, { "id": "e60f64a2394b1789a7dc", "model": "Claude-3.7-Sonnet-20250219", "organization": "Anthropic", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "Claude-3.7-Sonnet-20250219", "source_date": "2025-02-19", "report_url": "https://www.anthropic.com/news/claude-3-7-sonnet" }, { "id": "05bcc48ff6e5d6a81b45", "model": "GPT-4.1-2025-04-14", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "GPT-4.1-2025-04-14", "source_date": "2025-04-14", "report_url": "https://openai.com/index/gpt-4-1/" }, { "id": "6ef3cfbbec8c41741b3a", "model": "GPT-4.1-Mini-2025-04-14", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "GPT-4.1-Mini-2025-04-14", "source_date": "2025-04-14", "report_url": "https://openai.com/index/gpt-4-1/" }, { "id": "3a5e1b691f54958de124", "model": "Quasar-Alpha", "organization": "", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "Quasar-Alpha", "source_date": "2025-04-02", "report_url": "https://openrouter.ai/openrouter/quasar-alpha" }, { "id": "74edc4377bcf531b414e", "model": "Grok-3-Mini-Beta (temperature=1, reasoning=low)", "organization": "xAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.311, "raw_score": 31.1, "raw_scale": 100, "source_record": "Grok-3-Mini-Beta (temperature=1, reasoning=low)", "source_date": "2025-04-11", "report_url": "https://x.ai/news/grok-3" }, { "id": "d92b65fc790e8cbbac17", "model": "o3-mini-2025-01-31 (temperature=1, reasoning=low)", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.311, "raw_score": 31.1, "raw_scale": 100, "source_record": "o3-mini-2025-01-31 (temperature=1, reasoning=low)", "source_date": "2025-01-31", "report_url": "https://openai.com/index/openai-o3-mini/" }, { "id": "175ebfcba6df18d18674", "model": "Optimus-Alpha", "organization": "", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.304, "raw_score": 30.4, "raw_scale": 100, "source_record": "Optimus-Alpha", "source_date": "2025-04-10", "report_url": "https://openrouter.ai/openrouter/optimus-alpha" }, { "id": "47956ebc5472be0dba2b", "model": "Athene-V2-Agent", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Athene-V2-Agent", "source_date": "2024-11-14", "report_url": "https://huggingface.co/Nexusflow/Athene-V2-Agent" }, { "id": "f7d79f342a93114cdb2a", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "DeepSeek-R1", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1" }, { "id": "29e11ffd28bd1cf04b55", "model": "Gemini-2.5-Pro-Exp-03-25", "organization": "Google", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Gemini-2.5-Pro-Exp-03-25", "source_date": "2025-03-25", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/" }, { "id": "b21dbf0e06c7cc54d397", "model": "QwQ-32B (w/ Reasoning)", "organization": "", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "QwQ-32B (w/ Reasoning)", "source_date": "2025-03-06", "report_url": "https://huggingface.co/Qwen/QwQ-32B" }, { "id": "d469b608278b779018bf", "model": "o1-2024-12-17 (temperature=1, reasoning=low)", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "o1-2024-12-17 (temperature=1, reasoning=low)", "source_date": "2024-12-17", "report_url": "https://openai.com/o1/" }, { "id": "6c9f0d592b9ed2724c14", "model": "GPT-4-Turbo-2024-04-09", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "GPT-4-Turbo-2024-04-09", "source_date": "2024-04-09", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "77b7de2e585e927cf236", "model": "Qwen2.5-Max", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "Qwen2.5-Max", "source_date": "2025-01-28", "report_url": "https://qwenlm.github.io/blog/qwen2.5-max/" }, { "id": "4973eaa8e3ba92b1a901", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "DeepSeek-V3", "source_date": "2024-12-26", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3" }, { "id": "8a90f59cf488785f0113", "model": "GPT-4.1-Nano-2025-04-14", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "GPT-4.1-Nano-2025-04-14", "source_date": "2025-04-14", "report_url": "https://openai.com/index/gpt-4-1/" }, { "id": "97d06434c085ed09497b", "model": "Llama-3.3-70B-Instruct", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "Llama-3.3-70B-Instruct", "source_date": "2024-12-19", "report_url": "https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct" }, { "id": "50e6e7b80a960685f1bf", "model": "o1-2024-12-17 (temperature=1, reasoning=medium)", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "o1-2024-12-17 (temperature=1, reasoning=medium)", "source_date": "2024-12-17", "report_url": "https://openai.com/o1/" }, { "id": "a230375265ad2bb52c0f", "model": "DeepSeek-V3-0324", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "DeepSeek-V3-0324", "source_date": "2025-03-24", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324" }, { "id": "1327a2256ab190d453a4", "model": "GPT-4o-2024-11-20", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "GPT-4o-2024-11-20", "source_date": "2024-11-20", "report_url": "https://openai.com/gpt-4o/" }, { "id": "01ef0a20735eb6761f64", "model": "Gemini-Exp-1206", "organization": "Google", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "Gemini-Exp-1206", "source_date": "2024-12-06", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "48b9f4b337c8e379a4d4", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "Llama-4-Maverick", "source_date": "2025-04-05", "report_url": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct" }, { "id": "0a1f27c38a7a1134eb67", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "Qwen2.5-Coder-32B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct" }, { "id": "ae3e8b65664983047b75", "model": "o1-Mini-2024-09-12 (temperature=1)", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "o1-Mini-2024-09-12 (temperature=1)", "source_date": "2024-09-12", "report_url": "https://openai.com/index/introducing-openai-o1-preview/" }, { "id": "737aa6ae38a6f6aca30a", "model": "Athene-V2-Chat", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "Athene-V2-Chat", "source_date": "2024-11-14", "report_url": "https://huggingface.co/Nexusflow/Athene-V2-Chat" }, { "id": "09151d438cdc834ab3f3", "model": "DeepSeek-V2.5-1210", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "DeepSeek-V2.5-1210", "source_date": "2024-12-10" }, { "id": "5daa6936890763950805", "model": "Gemini-1.5-Pro-Exp-0827", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-Exp-0827", "source_date": "2024-08-27", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "39356b04593af94927c7", "model": "Gemini-2.0-Pro-Exp-02-05", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "Gemini-2.0-Pro-Exp-02-05", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "fcc7bb8c70dc67f1caaa", "model": "Grok-3-Beta", "organization": "xAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "Grok-3-Beta", "source_date": "2025-04-11", "report_url": "https://x.ai/news/grok-3" }, { "id": "f0af06aac84ffa2ce581", "model": "Gemini-Exp-1114", "organization": "Google", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.264, "raw_score": 26.4, "raw_scale": 100, "source_record": "Gemini-Exp-1114", "source_date": "2024-11-14", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "4a32de71d2358d01473e", "model": "Sky-T1-32B-Preview", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.264, "raw_score": 26.4, "raw_scale": 100, "source_record": "Sky-T1-32B-Preview", "source_date": "2025-01-12", "report_url": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Preview" }, { "id": "36243eb88a6b777df23c", "model": "ChatGPT-4o-latest-20250129", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.257, "raw_score": 25.7, "raw_scale": 100, "source_record": "ChatGPT-4o-latest-20250129", "source_date": "2025-01-29", "report_url": "https://chat.openai.com/" }, { "id": "a0a9e2fd09d658cbf6f3", "model": "Claude-3.5-Haiku-20241022", "organization": "Anthropic", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.257, "raw_score": 25.7, "raw_scale": 100, "source_record": "Claude-3.5-Haiku-20241022", "source_date": "2024-10-22", "report_url": "https://claude.ai/" }, { "id": "b662be56ed2b70e20dc0", "model": "Claude-3.5-Sonnet-20240620", "organization": "Anthropic", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.257, "raw_score": 25.7, "raw_scale": 100, "source_record": "Claude-3.5-Sonnet-20240620", "source_date": "2024-06-20", "report_url": "https://claude.ai/" }, { "id": "0e9ec69535f16dabc818", "model": "Claude-3.5-Sonnet-20241022", "organization": "Anthropic", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.257, "raw_score": 25.7, "raw_scale": 100, "source_record": "Claude-3.5-Sonnet-20241022", "source_date": "2024-10-22", "report_url": "https://claude.ai/" }, { "id": "e2eb3a5beb006734475f", "model": "DeepSeek-Coder-V2-Instruct (2024-07-24)", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.257, "raw_score": 25.7, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct (2024-07-24)", "source_date": "2024-07-24", "report_url": "https://www.deepseek.com/" }, { "id": "466c60dae478cf162b3d", "model": "Sky-T1-32B-Flash", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.257, "raw_score": 25.7, "raw_scale": 100, "source_record": "Sky-T1-32B-Flash", "source_date": "2025-01-12", "report_url": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Flash" }, { "id": "28c8e7619670fca9aa09", "model": "DeepSeek-V2-Chat (2024-06-28)", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "DeepSeek-V2-Chat (2024-06-28)", "source_date": "2024-06-28", "report_url": "https://www.deepseek.com/" }, { "id": "79448501633ce100dfdc", "model": "GPT-4o-2024-05-13", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "GPT-4o-2024-05-13", "source_date": "2024-05-13", "report_url": "https://openai.com/index/hello-gpt-4o/" }, { "id": "30e4e0c28d9a83f9eb35", "model": "GPT-4o-2024-08-06", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "GPT-4o-2024-08-06", "source_date": "2024-08-06", "report_url": "https://openai.com/index/introducing-structured-outputs-in-the-api/" }, { "id": "3f3248b803702533f6ad", "model": "Gemini-1.5-Pro-Exp-0801", "organization": "Google", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-Exp-0801", "source_date": "2024-08-01", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "6ab0c7f3af37f5d93de4", "model": "Gemma-3-27B-Instruct", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "Gemma-3-27B-Instruct", "source_date": "2025-03-12", "report_url": "https://huggingface.co/google/gemma-3-27b-it" }, { "id": "f0311033bb98d0757277", "model": "QwQ-32B-Preview", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "QwQ-32B-Preview", "source_date": "2024-11-28", "report_url": "https://huggingface.co/Qwen/QwQ-32B-Preview" }, { "id": "531cb4f68416b94fca45", "model": "DeepSeek-Coder-V2-Instruct", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.243, "raw_score": 24.3, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct" }, { "id": "5172eb82597f8ce3ceaf", "model": "Gemini-2.0-Flash-Thinking-Exp-01-21", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.243, "raw_score": 24.3, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Thinking-Exp-01-21", "source_date": "2025-01-21", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "f66cc0642e73ce693351", "model": "Gemini-2.0-Flash-Thinking-Exp-1219", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.243, "raw_score": 24.3, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Thinking-Exp-1219", "source_date": "2024-12-19", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "35a07cb26cdf3628e3a4", "model": "Gemini-Exp-1121", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.243, "raw_score": 24.3, "raw_scale": 100, "source_record": "Gemini-Exp-1121", "source_date": "2024-11-21", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "1cec437ecaa9e7f35384", "model": "Phi-4", "organization": "Microsoft", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.243, "raw_score": 24.3, "raw_scale": 100, "source_record": "Phi-4", "source_date": "2024-12-13", "report_url": "https://huggingface.co/microsoft/phi-4" }, { "id": "0ba9009f78dccd01f469", "model": "DeepSeek-R1-Distill-Qwen-32B", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-32B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" }, { "id": "583c1909ceca9bf51b86", "model": "GPT-4o-mini-2024-07-18", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "GPT-4o-mini-2024-07-18", "source_date": "2024-07-18", "report_url": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/" }, { "id": "c331d0495f7d217dc343", "model": "Gemini-2.0-Flash-001", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-001", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "008df1b6af1152839d9c", "model": "Grok-3-Mini-Beta (temperature=1, reasoning=high)", "organization": "xAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "Grok-3-Mini-Beta (temperature=1, reasoning=high)", "source_date": "2025-04-11", "report_url": "https://x.ai/news/grok-3" }, { "id": "772173e1af502d9b4b99", "model": "Grok-Beta", "organization": "xAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "Grok-Beta", "source_date": "2024-03-17", "report_url": "https://grok.com/" }, { "id": "c48068679d990e503b8f", "model": "Qwen2.5-Coder-14B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "Qwen2.5-Coder-14B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct" }, { "id": "f21b4930dbaaaa40c17c", "model": "DeepSeek-V2.5", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.23, "raw_score": 23.0, "raw_scale": 100, "source_record": "DeepSeek-V2.5", "source_date": "2024-09-18", "report_url": "https://www.deepseek.com/" }, { "id": "b9b4bad7f3f914d4700b", "model": "Gemini-2.0-Flash-Exp", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.23, "raw_score": 23.0, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Exp", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "55daabb935f5211c65ce", "model": "Llama-3.1-70B-Instruct", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.23, "raw_score": 23.0, "raw_scale": 100, "source_record": "Llama-3.1-70B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct" }, { "id": "e02b40d480ef4ff965b8", "model": "o1-Preview-2024-09-12 (temperature=1)", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.23, "raw_score": 23.0, "raw_scale": 100, "source_record": "o1-Preview-2024-09-12 (temperature=1)", "source_date": "2024-09-12", "report_url": "https://openai.com/index/introducing-openai-o1-preview/" }, { "id": "30b22f7eaedf70c07a8d", "model": "Claude-3-Opus-20240229", "organization": "Anthropic", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "Claude-3-Opus-20240229", "source_date": "2024-02-29", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "a3bf0e562269986eeb52", "model": "Gemini-1.5-Flash-API-0514", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "Gemini-1.5-Flash-API-0514", "source_date": "2024-05-14", "report_url": "https://deepmind.google/technologies/gemini/flash/" }, { "id": "191e36d7c9e59ab973b8", "model": "Llama-3-70B-Instruct", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "Llama-3-70B-Instruct", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct" }, { "id": "35bcd780c5bc708f12ee", "model": "Llama-3.1-405B-Instruct", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "Llama-3.1-405B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct" }, { "id": "d9be016a0671b4df8089", "model": "Mistral-Large-Instruct-2407", "organization": "Mistral AI", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "Mistral-Large-Instruct-2407", "source_date": "2024-07-24", "report_url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407" }, { "id": "b9b6acf62918ca57f5d9", "model": "Qwen2.5-32B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "Qwen2.5-32B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct" }, { "id": "0ffe809b06eb35a62f7d", "model": "DeepSeek-V2-Chat", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.21600000000000003, "raw_score": 21.6, "raw_scale": 100, "source_record": "DeepSeek-V2-Chat", "source_date": "2024-04-06", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat" }, { "id": "9bf57ffa8a433335d160", "model": "Mistral-Small-24B-Instruct-2501", "organization": "Mistral AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.21600000000000003, "raw_score": 21.6, "raw_scale": 100, "source_record": "Mistral-Small-24B-Instruct-2501", "source_date": "2025-01-31", "report_url": "https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501" }, { "id": "972a302e182b812fc2ec", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.21600000000000003, "raw_score": 21.6, "raw_scale": 100, "source_record": "Qwen2.5-72B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct" }, { "id": "08683bbc96ba70cd61e3", "model": "DeepSeek-R1-Distill-Qwen-14B", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-14B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B" }, { "id": "995765c13d4f27521c03", "model": "Dracarys-Llama-3.1-70B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "Dracarys-Llama-3.1-70B-Instruct", "source_date": "2024-04-23", "report_url": "https://huggingface.co/abacusai/Dracarys-Llama-3.1-70B-Instruct" }, { "id": "252556febaf908e90bd9", "model": "Gemini-1.5-Pro-002", "organization": "Google", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-002", "source_date": "2024-09-25", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "1c5251e09c83d24fbea0", "model": "Llama-3.1-Nemotron-70B-Instruct", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "Llama-3.1-Nemotron-70B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF" }, { "id": "2039b67a32ea922cfcd5", "model": "Qwen2.5-14B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "Qwen2.5-14B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct" }, { "id": "88699baaa38a9adf9be3", "model": "Claude-3-Sonnet-20240229", "organization": "Anthropic", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.203, "raw_score": 20.3, "raw_scale": 100, "source_record": "Claude-3-Sonnet-20240229", "source_date": "2024-02-29", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "4a114af99c8d3b80e971", "model": "DeepSeek-R1-Distill-Llama-70B", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.203, "raw_score": 20.3, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Llama-70B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B" }, { "id": "aabe2216f3ad8fe8bf24", "model": "Hermes-2-Theta-Llama-3-70B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.203, "raw_score": 20.3, "raw_scale": 100, "source_record": "Hermes-2-Theta-Llama-3-70B", "source_date": "2024-06-24", "report_url": "https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-70B" }, { "id": "d01d5d37cd200a83a4d1", "model": "Qwen2.5-Coder-7B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.203, "raw_score": 20.3, "raw_scale": 100, "source_record": "Qwen2.5-Coder-7B-Instruct", "source_date": "2024-11-12", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct" }, { "id": "28557ddb6a6f5e08a2bd", "model": "Gemini-1.5-Flash-Exp-0827", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.196, "raw_score": 19.6, "raw_scale": 100, "source_record": "Gemini-1.5-Flash-Exp-0827", "source_date": "2024-08-27", "report_url": "https://deepmind.google/technologies/gemini/flash/" }, { "id": "a15e5ef8fcf830c14ff1", "model": "Gemini-1.5-Pro-API-0514", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.196, "raw_score": 19.6, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-API-0514", "source_date": "2024-05-14", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "24cbe07bde6b4936460a", "model": "Gemini-2.0-Flash-Lite-Preview-02-05", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.196, "raw_score": 19.6, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Lite-Preview-02-05", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "94f34ece2cd608881244", "model": "ReflectionCoder-DS-33B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.196, "raw_score": 19.6, "raw_scale": 100, "source_record": "ReflectionCoder-DS-33B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-33B" }, { "id": "457d7f85397735e8f881", "model": "CodeGeex4-All-9B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.18899999999999997, "raw_score": 18.9, "raw_scale": 100, "source_record": "CodeGeex4-All-9B", "source_date": "2024-07-05", "report_url": "https://huggingface.co/THUDM/codegeex4-all-9b" }, { "id": "db1c1e11f754025a1f92", "model": "CodeQwen1.5-7B-Chat", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.18899999999999997, "raw_score": 18.9, "raw_scale": 100, "source_record": "CodeQwen1.5-7B-Chat", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat" }, { "id": "df56bf427cd5a73f58d8", "model": "Mixtral-8x22B-Instruct", "organization": "Mistral AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.18899999999999997, "raw_score": 18.9, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct", "source_date": "2024-04-17", "report_url": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1" }, { "id": "9fa936c7f7eef438953b", "model": "DeepCoder-14B-Preview", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "DeepCoder-14B-Preview", "source_date": "2025-04-09", "report_url": "https://huggingface.co/agentica-org/DeepCoder-14B-Preview" }, { "id": "1efbd962693289b4c69f", "model": "Dracarys-72B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "Dracarys-72B-Instruct", "source_date": "2024-04-23", "report_url": "https://huggingface.co/abacusai/Dracarys-72B-Instruct" }, { "id": "7020a0f966a0ef1931fb", "model": "GPT-3.5-Turbo-0125", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "GPT-3.5-Turbo-0125", "source_date": "2024-01-25", "report_url": "https://openai.com/index/new-embedding-models-and-api-updates" }, { "id": "41d099c2942e37049603", "model": "Llama-3-70B-Synthia-v3.5", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "Llama-3-70B-Synthia-v3.5", "source_date": "2024-05-27", "report_url": "https://huggingface.co/migtissera/Llama-3-70B-Synthia-v3.5" }, { "id": "a06e8815c7b78da345b5", "model": "Nxcode-CQ-7B-Orpo", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "Nxcode-CQ-7B-Orpo", "source_date": "2024-04-25", "report_url": "https://huggingface.co/NTQAI/Nxcode-CQ-7B-orpo" }, { "id": "691f84511d0ec15341c5", "model": "OpenCoder-8B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "OpenCoder-8B-Instruct", "source_date": "2024-11-09", "report_url": "https://huggingface.co/infly/OpenCoder-8B-Instruct" }, { "id": "880826db1a9c61240b8d", "model": "DeepSeek-Coder-33B-Instruct", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.17600000000000002, "raw_score": 17.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct" }, { "id": "31db15521e03453ab249", "model": "Gemma-2-27B-Instruct", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.17600000000000002, "raw_score": 17.6, "raw_scale": 100, "source_record": "Gemma-2-27B-Instruct", "source_date": "2024-06-19", "report_url": "https://huggingface.co/google/gemma-2-27b-it" }, { "id": "3baa03ec31c5ad034912", "model": "Phi-3.1-Mini-128K-Instruct", "organization": "Microsoft", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.17600000000000002, "raw_score": 17.6, "raw_scale": 100, "source_record": "Phi-3.1-Mini-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct" }, { "id": "2d906b4023f42a6866ad", "model": "WhiteRabbitNeo-33B-v1.5", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.17600000000000002, "raw_score": 17.6, "raw_scale": 100, "source_record": "WhiteRabbitNeo-33B-v1.5", "source_date": "2024-02-10", "report_url": "https://huggingface.co/WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5" }, { "id": "82ee262512a54a309c96", "model": "Claude-3-Haiku-20240307", "organization": "Anthropic", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.16899999999999998, "raw_score": 16.9, "raw_scale": 100, "source_record": "Claude-3-Haiku-20240307", "source_date": "2024-03-07", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "1a2aca7e452afb3aeff3", "model": "Codestral-22B-v0.1", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.16899999999999998, "raw_score": 16.9, "raw_scale": 100, "source_record": "Codestral-22B-v0.1", "source_date": "2024-05-23", "report_url": "https://huggingface.co/mistralai/Codestral-22B-v0.1" }, { "id": "0556e68fa458808a52fb", "model": "DeepCoder-14B-Preview (w/ Reasoning, 64k tokens, temperature=0.6)", "organization": "", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.16899999999999998, "raw_score": 16.9, "raw_scale": 100, "source_record": "DeepCoder-14B-Preview (w/ Reasoning, 64k tokens, temperature=0.6)", "source_date": "2025-04-09", "report_url": "https://huggingface.co/agentica-org/DeepCoder-14B-Preview" }, { "id": "fb257208509ac862fac3", "model": "Llama-4-Scout", "organization": "Meta", "configuration": "Hard-Instruct; prefill=False; prompted=True", "score": 0.16899999999999998, "raw_score": 16.9, "raw_scale": 100, "source_record": "Llama-4-Scout", "source_date": "2025-04-05", "report_url": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct" }, { "id": "d1dafd75f9b06dcff96e", "model": "Phi-3.5-Mini-Instruct", "organization": "Microsoft", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.16899999999999998, "raw_score": 16.9, "raw_scale": 100, "source_record": "Phi-3.5-Mini-Instruct", "source_date": "2024-08-21", "report_url": "https://huggingface.co/microsoft/Phi-3.5-mini-instruct" }, { "id": "25841a38654c0d8cc1fa", "model": "DeepSeek-Coder-V2-Lite-Instruct", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.162, "raw_score": 16.2, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Lite-Instruct", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct" }, { "id": "b452697db8b6e46ac420", "model": "Athene-70B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "Athene-70B", "source_date": "2024-07-20", "report_url": "https://huggingface.co/Nexusflow/Athene-70B" }, { "id": "eb2e685465bc054d6df0", "model": "AutoCoder", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "AutoCoder", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder" }, { "id": "dbe1e7661453b52708e1", "model": "Yi-Large", "organization": "01.AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "Yi-Large", "source_date": "2024-05-13", "report_url": "https://www.lingyiwanwu.com/" }, { "id": "a208d7a5691d17f7397a", "model": "GPT-4-0613", "organization": "OpenAI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "GPT-4-0613", "source_date": "2024-06-13", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "f3313792cf5138dd4ea3", "model": "Hermes-2-Pro-Llama-3-70B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "Hermes-2-Pro-Llama-3-70B", "source_date": "2024-06-27", "report_url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-70B" }, { "id": "ce163386751f026ab2c1", "model": "Qwen2-72B-Chat", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "Qwen2-72B-Chat", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-72B-Chat" }, { "id": "f78d9deb8a19887b291a", "model": "Tess-v2.5.2-Qwen2-72B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "Tess-v2.5.2-Qwen2-72B", "source_date": "2024-07-18", "report_url": "https://huggingface.co/migtissera/Tess-v2.5.2-Qwen2-72B" }, { "id": "b5bfc24428b007ff6475", "model": "Granite-Code-34B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "Granite-Code-34B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-34b-code-instruct" }, { "id": "3cdc2c68a81d341ccbae", "model": "InverseCoder-CL-7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "InverseCoder-CL-7B", "source_date": "2024-07-08", "report_url": "https://huggingface.co/wyt2000/InverseCoder-CL-7B" }, { "id": "ed5a2c885a5a0e38e2b7", "model": "WarriorCoder-6.7B (Reproduced)", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "WarriorCoder-6.7B (Reproduced)", "source_date": "2025-02-18", "report_url": "https://arxiv.org/abs/2412.17395" }, { "id": "0424a91abbd6ada2af5b", "model": "C4AI-Command-R-08-2024", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "C4AI-Command-R-08-2024", "source_date": "2024-08-30", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-08-2024" }, { "id": "505064d9efe3613d0dbb", "model": "Command R+", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "Command R+", "source_date": "2024-04-04", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-plus" }, { "id": "f9b45bd93750ea1d1681", "model": "Llama-3.1-8B-Instruct", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "Llama-3.1-8B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct" }, { "id": "d1ef871a823b03b69612", "model": "Magicoder-S-DS-6.7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "Magicoder-S-DS-6.7B", "source_date": "2024-12-04", "report_url": "https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B" }, { "id": "fb1399a69666a126eaff", "model": "OpenCodeInterpreter-DS-6.7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-6.7B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B" }, { "id": "fc4539651aa3cda0fbdc", "model": "Codestral-Mamba", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "Codestral-Mamba", "source_date": "2024-07-16", "report_url": "https://huggingface.co/mistralai/mamba-codestral-7B-v0.1" }, { "id": "b936f565b01b32eb6fd1", "model": "InverseCoder-CL-13B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "InverseCoder-CL-13B", "source_date": "2024-07-08", "report_url": "https://huggingface.co/wyt2000/InverseCoder-CL-13B" }, { "id": "0667522f2464c8f617db", "model": "Mistral-Small-2402", "organization": "Mistral AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "Mistral-Small-2402", "source_date": "2024-02-26", "report_url": "https://mistral.ai/news/mistral-large/" }, { "id": "c98656b9280aa90ed5ab", "model": "Phind-CodeLlama-34B-v2", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "Phind-CodeLlama-34B-v2", "source_date": "2023-08-25", "report_url": "https://huggingface.co/Phind/Phind-CodeLlama-34B-v2" }, { "id": "172fc6a41d0f943afeb2", "model": "WaveCoder-Ultra-6.7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "WaveCoder-Ultra-6.7B", "source_date": "2023-12-26", "report_url": "https://huggingface.co/microsoft/wavecoder-ultra-6.7b" }, { "id": "39651f4435d5050c9212", "model": "Artigenz-Coder-DS-6.7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "Artigenz-Coder-DS-6.7B", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Artigenz/Artigenz-Coder-DS-6.7B" }, { "id": "3a8fc11881e03b27a72f", "model": "Qwen1.5-110B-Chat", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "Qwen1.5-110B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-110B-Chat" }, { "id": "4afdad5104fa1b40e87e", "model": "Qwen1.5-72B-Chat", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "Qwen1.5-72B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-72B-Chat" }, { "id": "b61019fb4a8a8670e461", "model": "Qwen2.5-7B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "Qwen2.5-7B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct" }, { "id": "1d2fea03400612233f90", "model": "StarCoder2-15B-Instruct-v0.1", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "StarCoder2-15B-Instruct-v0.1", "source_date": "2024-04-30", "report_url": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1" }, { "id": "e19fcc044ca2daf86af7", "model": "C4AI-Command-R-Plus-08-2024", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "C4AI-Command-R-Plus-08-2024", "source_date": "2024-08-30", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024" }, { "id": "b8f265c234cd3f272e73", "model": "CodeLlama-70B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "CodeLlama-70B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf" }, { "id": "98db673e5e6547a2035f", "model": "Granite-Code-20B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "Granite-Code-20B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-20b-code-instruct" }, { "id": "5dd235c408a54c2143af", "model": "Mistral-Nemo-12B-Instruct", "organization": "Mistral AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "Mistral-Nemo-12B-Instruct", "source_date": "2024-07-18", "report_url": "https://huggingface.co/nv-mistralai/Mistral-Nemo-12B-Instruct" }, { "id": "cd6f07fbcf06d98a1a81", "model": "ReflectionCoder-CL-34B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "ReflectionCoder-CL-34B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-34B" }, { "id": "e15aaa25c73230d63687", "model": "ReflectionCoder-DS-6.7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "ReflectionCoder-DS-6.7B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-6.7B" }, { "id": "93c7acd979f4b385ae10", "model": "Yi-Coder-9B-Chat", "organization": "01.AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "Yi-Coder-9B-Chat", "source_date": "2024-09-04", "report_url": "https://huggingface.co/01-ai/Yi-Coder-9B-Chat" }, { "id": "4ee69372c7e9a2209ddc", "model": "InverseCoder-DS-6.7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10800000000000001, "raw_score": 10.8, "raw_scale": 100, "source_record": "InverseCoder-DS-6.7B", "source_date": "2024-07-08", "report_url": "https://huggingface.co/wyt2000/InverseCoder-DS-6.7B" }, { "id": "dd07a3eb0ca36bde1336", "model": "Mistral-Small-Instruct-2409", "organization": "Mistral AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10800000000000001, "raw_score": 10.8, "raw_scale": 100, "source_record": "Mistral-Small-Instruct-2409", "source_date": "2024-09-18", "report_url": "https://huggingface.co/mistralai/Mistral-Small-Instruct-2409" }, { "id": "21dadbca66e9a80cac7b", "model": "Phi-3-Medium-128K-Instruct", "organization": "Microsoft", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10800000000000001, "raw_score": 10.8, "raw_scale": 100, "source_record": "Phi-3-Medium-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-medium-128k-instruct" }, { "id": "12f23f959927b2011e48", "model": "Phi-3-Small-128K-Instruct", "organization": "Microsoft", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10800000000000001, "raw_score": 10.8, "raw_scale": 100, "source_record": "Phi-3-Small-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-small-128k-instruct" }, { "id": "584fc8797f1014af0fda", "model": "DeepSeek-Coder-6.7B-Instruct", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10099999999999999, "raw_score": 10.1, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct" }, { "id": "3a33f63e651f14b9122a", "model": "Gemma-2-9B-Instruct", "organization": "Google", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10099999999999999, "raw_score": 10.1, "raw_scale": 100, "source_record": "Gemma-2-9B-Instruct", "source_date": "2024-06-19", "report_url": "https://huggingface.co/google/gemma-2-9b-it" }, { "id": "80b8ad04ff5542d82114", "model": "Qwen2-57B-A14B", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10099999999999999, "raw_score": 10.1, "raw_scale": 100, "source_record": "Qwen2-57B-A14B", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-57B-A14B" }, { "id": "e6e26b4b9e2582b38680", "model": "Qwen2-7B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10099999999999999, "raw_score": 10.1, "raw_scale": 100, "source_record": "Qwen2-7B-Instruct", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-7B-Instruct" }, { "id": "e61d28081296e928c7d9", "model": "ReflectionCoder-CL-7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10099999999999999, "raw_score": 10.1, "raw_scale": 100, "source_record": "ReflectionCoder-CL-7B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-7B" }, { "id": "789765d27879537523d7", "model": "Yi-1.5-9B-Chat", "organization": "01.AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.10099999999999999, "raw_score": 10.1, "raw_scale": 100, "source_record": "Yi-1.5-9B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-9B-Chat" }, { "id": "52f557045fc8516db57b", "model": "CodeLlama-13B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.095, "raw_score": 9.5, "raw_scale": 100, "source_record": "CodeLlama-13B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf" }, { "id": "47ab37520afe432df077", "model": "OpenCoder-1.5B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.095, "raw_score": 9.5, "raw_scale": 100, "source_record": "OpenCoder-1.5B-Instruct", "source_date": "2024-11-09", "report_url": "https://huggingface.co/infly/OpenCoder-1.5B-Instruct" }, { "id": "d34c081bee7a70214e79", "model": "Qwen1.5-32B-Chat", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.095, "raw_score": 9.5, "raw_scale": 100, "source_record": "Qwen1.5-32B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-32B-Chat" }, { "id": "f88cc9c0f66d0b1d4001", "model": "AutoCoder-QW-7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "AutoCoder-QW-7B", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder_QW_7B" }, { "id": "9a43355adba7fdd40f0b", "model": "AutoCoder-S-6.7B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "AutoCoder-S-6.7B", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder_S_6.7B" }, { "id": "3fbe41841dfcda2d7566", "model": "CodeLlama-34B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "CodeLlama-34B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf" }, { "id": "25647cdb7148cc992141", "model": "OpenChat-3.6-8B-20240522", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "OpenChat-3.6-8B-20240522", "source_date": "2024-05-22", "report_url": "https://huggingface.co/openchat/openchat-3.6-8b-20240522" }, { "id": "65ef3b93e8c26ddc5013", "model": "Yi-1.5-34B-Chat", "organization": "01.AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "Yi-1.5-34B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-34B-Chat" }, { "id": "18d08a65d81ce38b6d84", "model": "CodeGemma-7B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.07400000000000001, "raw_score": 7.4, "raw_scale": 100, "source_record": "CodeGemma-7B-Instruct", "source_date": "2024-04-10", "report_url": "https://huggingface.co/google/codegemma-7b-it" }, { "id": "c886958321a7ab6bde93", "model": "Granite-3.0-8B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.07400000000000001, "raw_score": 7.4, "raw_scale": 100, "source_record": "Granite-3.0-8B-Instruct", "source_date": "2024-10-21", "report_url": "https://huggingface.co/ibm-granite/granite-3.0-8b-instruct" }, { "id": "4a209c72393be25357eb", "model": "Phi-3-Mini-128K-Instruct", "organization": "Microsoft", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.07400000000000001, "raw_score": 7.4, "raw_scale": 100, "source_record": "Phi-3-Mini-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct" }, { "id": "cce44d106c4b3990eee1", "model": "Llama-3-8B-Instruct", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.068, "raw_score": 6.8, "raw_scale": 100, "source_record": "Llama-3-8B-Instruct", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct" }, { "id": "cc9e2326b7c1594793c8", "model": "Llama-3.2-3B-Instruct", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.054000000000000006, "raw_score": 5.4, "raw_scale": 100, "source_record": "Llama-3.2-3B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct" }, { "id": "ec7337d7fc279611920e", "model": "OpenCodeInterpreter-DS-1.3B", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.054000000000000006, "raw_score": 5.4, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-1.3B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-1.3B" }, { "id": "6629c1aa64afd2993f3d", "model": "Qwen2.5-Coder-1.5B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.054000000000000006, "raw_score": 5.4, "raw_scale": 100, "source_record": "Qwen2.5-Coder-1.5B-Instruct", "source_date": "2024-11-12", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct" }, { "id": "f0039c47cca363813d23", "model": "Mistral-7B-Instruct-v0.3", "organization": "Mistral AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.047, "raw_score": 4.7, "raw_scale": 100, "source_record": "Mistral-7B-Instruct-v0.3", "source_date": "2024-05-22", "report_url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3" }, { "id": "3f74609b2e0f679e0594", "model": "Mistral-Large-2402", "organization": "Mistral AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.040999999999999995, "raw_score": 4.1, "raw_scale": 100, "source_record": "Mistral-Large-2402", "source_date": "2024-02-26", "report_url": "https://mistral.ai/news/mistral-large/" }, { "id": "1cf7cd4c4f5fe21df135", "model": "Yi-1.5-6B-Chat", "organization": "01.AI", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.040999999999999995, "raw_score": 4.1, "raw_scale": 100, "source_record": "Yi-1.5-6B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-6B-Chat" }, { "id": "27a797d5d20833c33e68", "model": "CodeLlama-7B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.034, "raw_score": 3.4, "raw_scale": 100, "source_record": "CodeLlama-7B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf" }, { "id": "921aca304e1a6b4bcc32", "model": "DeepSeek-Coder-1.3B-Instruct", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.034, "raw_score": 3.4, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct" }, { "id": "8c1dd3ea17320962ada3", "model": "DeepSeek-R1-Distill-Qwen-7B", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.034, "raw_score": 3.4, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-7B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" }, { "id": "74c47691997bd779916a", "model": "InternLM2.5-7B-Chat", "organization": "Shanghai AI Laboratory", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.034, "raw_score": 3.4, "raw_scale": 100, "source_record": "InternLM2.5-7B-Chat", "source_date": "2024-07-03", "report_url": "https://huggingface.co/internlm/internlm2_5-7b-chat" }, { "id": "22fc60302c3cd4a2fd1f", "model": "DeepSeek-R1-Distill-Llama-8B", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.02, "raw_score": 2.0, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Llama-8B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B" }, { "id": "d4bbc5246f09eeeed7bd", "model": "Granite-3.0-2B-Instruct", "organization": "", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.02, "raw_score": 2.0, "raw_scale": 100, "source_record": "Granite-3.0-2B-Instruct", "source_date": "2024-10-21", "report_url": "https://huggingface.co/ibm-granite/granite-3.0-2b-instruct" }, { "id": "54f0332f9b7255ed0edc", "model": "Qwen2.5-1.5B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.013999999999999999, "raw_score": 1.4, "raw_scale": 100, "source_record": "Qwen2.5-1.5B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct" }, { "id": "4a3bd480d04ad502597b", "model": "Llama-3.2-1B-Instruct", "organization": "Meta", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.006999999999999999, "raw_score": 0.7, "raw_scale": 100, "source_record": "Llama-3.2-1B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct" }, { "id": "2d399ef44ac721206ab9", "model": "Qwen2.5-0.5B-Instruct", "organization": "Alibaba", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.006999999999999999, "raw_score": 0.7, "raw_scale": 100, "source_record": "Qwen2.5-0.5B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct" }, { "id": "4da4589d256fc2bb8534", "model": "DeepSeek-R1-Distill-Qwen-1.5B", "organization": "DeepSeek", "configuration": "Hard-Instruct; prefill=True; prompted=True", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-1.5B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" } ] }, { "id": "humaneval", "name": "HumanEval", "group": "编程", "unit": "%", "metric": "pass@1", "protocol": { "en": "EvalPlus published pass@1. HumanEval has 164 tasks; MBPP uses the 399-task EvalPlus subset. Base tests and expanded + tests remain separate.", "zh": "EvalPlus 发布的 pass@1。HumanEval 为 164 道任务；MBPP 使用 EvalPlus 的 399 道题子集。原始测试与扩展 + 测试分别展示。" }, "source": { "data_url": "https://evalplus.github.io/results.json", "sha256": "e6837dc3b94f32cb1da0ced012e52edc4a722007b684df5b63da029bbd3b46d2", "retrieved_at": "2026-11-01T16:39:18.103888+00:00", "name": "EvalPlus", "url": "https://evalplus.github.io/leaderboard.html", "selector": "pass@1.humaneval", "status": "current_snapshot", "kind": "evaluator" }, "results": [ { "id": "2a45d8a080de4798ae96", "model": "O1 Mini (Sept 2024)", "organization": "", "configuration": "Chat prompted", "score": 0.963, "raw_score": 96.3, "raw_scale": 100, "source_record": "O1 Mini (Sept 2024)" }, { "id": "084039330e5ae9df8fdd", "model": "O1 Preview (Sept 2024)", "organization": "", "configuration": "Chat prompted", "score": 0.963, "raw_score": 96.3, "raw_scale": 100, "source_record": "O1 Preview (Sept 2024)" }, { "id": "4ea990b3089f21fc6dae", "model": "GPT 4o (Aug 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.927, "raw_score": 92.7, "raw_scale": 100, "source_record": "GPT 4o (Aug 2024)" }, { "id": "0a1f27c38a7a1134eb67", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Chat prompted", "score": 0.9209999999999999, "raw_score": 92.1, "raw_scale": 100, "source_record": "Qwen2.5-Coder-32B-Instruct" }, { "id": "53f47496f476362a7ce1", "model": "DeepSeek-V3 (Nov 2024)", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.915, "raw_score": 91.5, "raw_scale": 100, "source_record": "DeepSeek-V3 (Nov 2024)" }, { "id": "b8c790bc669412a2379b", "model": "DeepSeek-V2.5 (Nov 2024)", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "DeepSeek-V2.5 (Nov 2024)" }, { "id": "b392daa9ccf08c8e805c", "model": "GPT-4-Turbo (April 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "GPT-4-Turbo (April 2024)" }, { "id": "0b7407a75379444a79fb", "model": "Gemini 1.5 Pro 002", "organization": "Google", "configuration": "Chat prompted", "score": 0.89, "raw_score": 89.0, "raw_scale": 100, "source_record": "Gemini 1.5 Pro 002" }, { "id": "db1cab9d6afa1cf84290", "model": "GPT 4o Mini (July 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.884, "raw_score": 88.4, "raw_scale": 100, "source_record": "GPT 4o Mini (July 2024)" }, { "id": "a139c9468fc1e37ea9ed", "model": "GPT-4 (May 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.884, "raw_score": 88.4, "raw_scale": 100, "source_record": "GPT-4 (May 2023)" }, { "id": "769fede65d4cf9903d11", "model": "Grok Beta", "organization": "xAI", "configuration": "Chat prompted", "score": 0.884, "raw_score": 88.4, "raw_scale": 100, "source_record": "Grok Beta" }, { "id": "1c46e296b7812dc418c8", "model": "Claude Sonnet 3.5 (June 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.872, "raw_score": 87.2, "raw_scale": 100, "source_record": "Claude Sonnet 3.5 (June 2024)" }, { "id": "531cb4f68416b94fca45", "model": "DeepSeek-Coder-V2-Instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.8540000000000001, "raw_score": 85.4, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct" }, { "id": "80d858ccbb48ab9ec53b", "model": "GPT-4-Turbo (Nov 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.8540000000000001, "raw_score": 85.4, "raw_scale": 100, "source_record": "GPT-4-Turbo (Nov 2023)" }, { "id": "db1c1e11f754025a1f92", "model": "CodeQwen1.5-7B-Chat", "organization": "", "configuration": "Chat prompted", "score": 0.835, "raw_score": 83.5, "raw_scale": 100, "source_record": "CodeQwen1.5-7B-Chat" }, { "id": "62e201fb8adbc72d49fa", "model": "claude-3-opus (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.8290000000000001, "raw_score": 82.9, "raw_scale": 100, "source_record": "claude-3-opus (Mar 2024)" }, { "id": "0fe26a52dd1b57b26968", "model": "Gemini 1.5 Flash 002", "organization": "Google", "configuration": "Chat prompted", "score": 0.823, "raw_score": 82.3, "raw_scale": 100, "source_record": "Gemini 1.5 Flash 002" }, { "id": "691f84511d0ec15341c5", "model": "OpenCoder-8B-Instruct", "organization": "", "configuration": "Chat prompted", "score": 0.8170000000000001, "raw_score": 81.7, "raw_scale": 100, "source_record": "OpenCoder-8B-Instruct" }, { "id": "90184c89f29471ceb2e3", "model": "DeepSeek-Coder-33B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.8109999999999999, "raw_score": 81.1, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-instruct" }, { "id": "1a2aca7e452afb3aeff3", "model": "Codestral-22B-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.799, "raw_score": 79.9, "raw_scale": 100, "source_record": "Codestral-22B-v0.1" }, { "id": "8b4c4e5236cfdb78aec9", "model": "WizardCoder-33B-V1.1", "organization": "", "configuration": "Chat prompted", "score": 0.799, "raw_score": 79.9, "raw_scale": 100, "source_record": "WizardCoder-33B-V1.1" }, { "id": "56928ca9ba87a6db3cc6", "model": "OpenCodeInterpreter-DS-33B", "organization": "", "configuration": "Chat prompted", "score": 0.7929999999999999, "raw_score": 79.3, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-33B" }, { "id": "12cb129efc0d3d8dcb57", "model": "Llama3-70B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.774, "raw_score": 77.4, "raw_scale": 100, "source_record": "Llama3-70B-instruct" }, { "id": "fb1399a69666a126eaff", "model": "OpenCodeInterpreter-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.774, "raw_score": 77.4, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-6.7B" }, { "id": "280581193c6a0c9a67dc", "model": "speechless-codellama-34B-v2.0", "organization": "", "configuration": "Chat prompted", "score": 0.774, "raw_score": 77.4, "raw_scale": 100, "source_record": "speechless-codellama-34B-v2.0" }, { "id": "44d35736cd73f5e16752", "model": "GPT-3.5-Turbo (Nov 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.768, "raw_score": 76.8, "raw_scale": 100, "source_record": "GPT-3.5-Turbo (Nov 2023)" }, { "id": "d1ef871a823b03b69612", "model": "Magicoder-S-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.768, "raw_score": 76.8, "raw_scale": 100, "source_record": "Magicoder-S-DS-6.7B" }, { "id": "1cfc65ad1da9ba081cb4", "model": "claude-3-haiku (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.768, "raw_score": 76.8, "raw_scale": 100, "source_record": "claude-3-haiku (Mar 2024)" }, { "id": "f7e707367b35c8316fa3", "model": "Mixtral-8x22B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.762, "raw_score": 76.2, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct-v0.1" }, { "id": "39651f4435d5050c9212", "model": "Artigenz-Coder-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.7559999999999999, "raw_score": 75.6, "raw_scale": 100, "source_record": "Artigenz-Coder-DS-6.7B" }, { "id": "eb111531fd091ddc90ae", "model": "DeepSeek-Coder-7B-instruct-v1.5", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.7559999999999999, "raw_score": 75.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-7B-instruct-v1.5" }, { "id": "566a6f6a778e843caa18", "model": "XwinCoder-34B", "organization": "", "configuration": "Chat prompted", "score": 0.7559999999999999, "raw_score": 75.6, "raw_scale": 100, "source_record": "XwinCoder-34B" }, { "id": "172fc6a41d0f943afeb2", "model": "WaveCoder-Ultra-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "WaveCoder-Ultra-6.7B" }, { "id": "d1a87920d704d9eb6a2f", "model": "databricks/dbrx-instruct", "organization": "", "configuration": "Chat prompted", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "databricks/dbrx-instruct" }, { "id": "93181f1d7ffc42d0ed49", "model": "DeepSeek-Coder-6.7B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.7440000000000001, "raw_score": 74.4, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-instruct" }, { "id": "1d58398274f62163ce53", "model": "code-millenials-34B", "organization": "", "configuration": "Chat prompted", "score": 0.7440000000000001, "raw_score": 74.4, "raw_scale": 100, "source_record": "code-millenials-34B" }, { "id": "84cf4ef9b52e43610ec4", "model": "starchat2-15b-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.738, "raw_score": 73.8, "raw_scale": 100, "source_record": "starchat2-15b-v0.1" }, { "id": "a9c0da8f899188336f08", "model": "GPT-3.5 (May 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.732, "raw_score": 73.2, "raw_scale": 100, "source_record": "GPT-3.5 (May 2023)" }, { "id": "d5be5926ead27aa7faa4", "model": "WizardCoder-Python-34B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.732, "raw_score": 73.2, "raw_scale": 100, "source_record": "WizardCoder-Python-34B-V1.0" }, { "id": "f5b8f9757571067b6982", "model": "OpenChat-3.5-7B-0106", "organization": "", "configuration": "Chat prompted", "score": 0.726, "raw_score": 72.6, "raw_scale": 100, "source_record": "OpenChat-3.5-7B-0106" }, { "id": "b8f265c234cd3f272e73", "model": "CodeLlama-70B-Instruct", "organization": "", "configuration": "Chat prompted", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "CodeLlama-70B-Instruct" }, { "id": "4052178124a22c33fce0", "model": "WhiteRabbitNeo-33B-v1", "organization": "", "configuration": "Chat prompted", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "WhiteRabbitNeo-33B-v1" }, { "id": "c98656b9280aa90ed5ab", "model": "Phind-CodeLlama-34B-v2", "organization": "", "configuration": "Completion", "score": 0.713, "raw_score": 71.3, "raw_scale": 100, "source_record": "Phind-CodeLlama-34B-v2" }, { "id": "3dc355840d04012b3892", "model": "speechless-coder-ds-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.713, "raw_score": 71.3, "raw_scale": 100, "source_record": "speechless-coder-ds-6.7B" }, { "id": "2141e7a005e32dd9e409", "model": "Magicoder-S-CL-7B", "organization": "", "configuration": "Chat prompted", "score": 0.7070000000000001, "raw_score": 70.7, "raw_scale": 100, "source_record": "Magicoder-S-CL-7B" }, { "id": "7d2e36d3604d93694e87", "model": "claude-3-sonnet (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.7070000000000001, "raw_score": 70.7, "raw_scale": 100, "source_record": "claude-3-sonnet (Mar 2024)" }, { "id": "8d512e3e8216a32abb97", "model": "Llama3.1-8B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.695, "raw_score": 69.5, "raw_scale": 100, "source_record": "Llama3.1-8B-instruct" }, { "id": "f12d2bd74dd1e3d0c0e5", "model": "Mistral Large (Mar 2024)", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.695, "raw_score": 69.5, "raw_scale": 100, "source_record": "Mistral Large (Mar 2024)" }, { "id": "85a84626a2de31308989", "model": "claude-2 (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.695, "raw_score": 69.5, "raw_scale": 100, "source_record": "claude-2 (Mar 2024)" }, { "id": "f47a7de678f154245c59", "model": "Gemini Pro 1.5", "organization": "Google", "configuration": "Chat prompted", "score": 0.6829999999999999, "raw_score": 68.3, "raw_scale": 100, "source_record": "Gemini Pro 1.5" }, { "id": "4afdad5104fa1b40e87e", "model": "Qwen1.5-72B-Chat", "organization": "Alibaba", "configuration": "Chat prompted", "score": 0.6829999999999999, "raw_score": 68.3, "raw_scale": 100, "source_record": "Qwen1.5-72B-Chat" }, { "id": "c8a626855515bd14b1b4", "model": "starcoder2-15b-instruct-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.677, "raw_score": 67.7, "raw_scale": 100, "source_record": "starcoder2-15b-instruct-v0.1" }, { "id": "a54728bce2fb3a3af23e", "model": "speechless-starcoder2-15b", "organization": "", "configuration": "Chat prompted", "score": 0.6709999999999999, "raw_score": 67.1, "raw_scale": 100, "source_record": "speechless-starcoder2-15b" }, { "id": "d405297a8cbc9798976d", "model": "DeepSeek-Coder-1.3B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.659, "raw_score": 65.9, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-instruct" }, { "id": "9e6e87afd361ba906c29", "model": "Code-290k-6.7B-Instruct", "organization": "", "configuration": "Chat prompted", "score": 0.6459999999999999, "raw_score": 64.6, "raw_scale": 100, "source_record": "Code-290k-6.7B-Instruct" }, { "id": "e475320e773d474a2943", "model": "Phi-3-mini-4k-instruct", "organization": "Microsoft", "configuration": "Chat prompted", "score": 0.6459999999999999, "raw_score": 64.6, "raw_scale": 100, "source_record": "Phi-3-mini-4k-instruct" }, { "id": "6398e7bd9d49b2534f97", "model": "Command-R+", "organization": "", "configuration": "Chat prompted", "score": 0.64, "raw_score": 64.0, "raw_scale": 100, "source_record": "Command-R+" }, { "id": "1276ae6f21eb1bd0bb29", "model": "dolphin-2.6-mixtral-8x7b", "organization": "", "configuration": "Chat prompted", "score": 0.64, "raw_score": 64.0, "raw_scale": 100, "source_record": "dolphin-2.6-mixtral-8x7b" }, { "id": "7e3a60ef3ae22f35efc4", "model": "Gemini Pro 1.0", "organization": "Google", "configuration": "Chat prompted", "score": 0.634, "raw_score": 63.4, "raw_scale": 100, "source_record": "Gemini Pro 1.0" }, { "id": "c75e7da3d9fe19708a6f", "model": "Llama3-8B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.616, "raw_score": 61.6, "raw_scale": 100, "source_record": "Llama3-8B-instruct" }, { "id": "3fdafb78c6f7d8c92289", "model": "codegemma-7b-it", "organization": "", "configuration": "Chat prompted", "score": 0.604, "raw_score": 60.4, "raw_scale": 100, "source_record": "codegemma-7b-it" }, { "id": "82367639fb7a6486bcf4", "model": "claude-instant-1 (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.573, "raw_score": 57.3, "raw_scale": 100, "source_record": "claude-instant-1 (Mar 2024)" }, { "id": "4e2a62e8daafafba2736", "model": "WizardCoder-15B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.5670000000000001, "raw_score": 56.7, "raw_scale": 100, "source_record": "WizardCoder-15B-V1.0" }, { "id": "53a0daaf8f346170de88", "model": "Code-13B", "organization": "", "configuration": "Chat prompted", "score": 0.561, "raw_score": 56.1, "raw_scale": 100, "source_record": "Code-13B" }, { "id": "3f0edba6458b50bd4cc1", "model": "speechless-starcoder2-7b", "organization": "", "configuration": "Chat prompted", "score": 0.561, "raw_score": 56.1, "raw_scale": 100, "source_record": "speechless-starcoder2-7b" }, { "id": "4907f74fca56a1e14b1a", "model": "CodeLlama-70B", "organization": "", "configuration": "Completion", "score": 0.555, "raw_score": 55.5, "raw_scale": 100, "source_record": "CodeLlama-70B" }, { "id": "baa147a39d546544863b", "model": "Code-33B", "organization": "", "configuration": "Chat prompted", "score": 0.5489999999999999, "raw_score": 54.9, "raw_scale": 100, "source_record": "Code-33B" }, { "id": "06d2430a53a2a51bb3b0", "model": "speechless-coding-7B-16k-tora", "organization": "", "configuration": "Chat prompted", "score": 0.5489999999999999, "raw_score": 54.9, "raw_scale": 100, "source_record": "speechless-coding-7B-16k-tora" }, { "id": "d2f6debe8f038a5a758b", "model": "OpenHermes-2.5-Code-290k-13B", "organization": "", "configuration": "Chat prompted", "score": 0.5429999999999999, "raw_score": 54.3, "raw_scale": 100, "source_record": "OpenHermes-2.5-Code-290k-13B" }, { "id": "273c3cc8d56437443071", "model": "CodeLlama-34B", "organization": "", "configuration": "Completion", "score": 0.518, "raw_score": 51.8, "raw_scale": 100, "source_record": "CodeLlama-34B" }, { "id": "bb051df20f0935ee2e27", "model": "CodeQwen1.5-7B", "organization": "", "configuration": "Completion", "score": 0.518, "raw_score": 51.8, "raw_scale": 100, "source_record": "CodeQwen1.5-7B" }, { "id": "ce783e5b1dae0a26079d", "model": "DeepSeek-Coder-33B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.512, "raw_score": 51.2, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-base" }, { "id": "674a9924e98b9b389a8d", "model": "WizardCoder-Python-7B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.506, "raw_score": 50.6, "raw_scale": 100, "source_record": "WizardCoder-Python-7B-V1.0" }, { "id": "2799abfaf8ad6d2a2646", "model": "phi-2-2.7B", "organization": "Microsoft", "configuration": "Completion", "score": 0.494, "raw_score": 49.4, "raw_scale": 100, "source_record": "phi-2-2.7B" }, { "id": "b7a8fb76da274a850160", "model": "Mistral-codealpaca-7B", "organization": "Mistral AI", "configuration": "Completion", "score": 0.48200000000000004, "raw_score": 48.2, "raw_scale": 100, "source_record": "Mistral-codealpaca-7B" }, { "id": "db6ffffa8f44dbfa1134", "model": "speechless-code-mistral-7B-v1.0", "organization": "", "configuration": "Chat prompted", "score": 0.48200000000000004, "raw_score": 48.2, "raw_scale": 100, "source_record": "speechless-code-mistral-7B-v1.0" }, { "id": "9f6a1b5fbe306eeea58f", "model": "DeepSeek-Coder-6.7B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.47600000000000003, "raw_score": 47.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-base" }, { "id": "88050667dacbe8ae23a8", "model": "MistralHermes-CodePro-7B-v1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.47600000000000003, "raw_score": 47.6, "raw_scale": 100, "source_record": "MistralHermes-CodePro-7B-v1" }, { "id": "f2b2f5ff73e7fc7a96c4", "model": "StarCoder2-15B", "organization": "", "configuration": "Completion", "score": 0.46299999999999997, "raw_score": 46.3, "raw_scale": 100, "source_record": "StarCoder2-15B" }, { "id": "0a800dac99fb938f31f2", "model": "Mixtral-8x7B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.451, "raw_score": 45.1, "raw_scale": 100, "source_record": "Mixtral-8x7B-Instruct-v0.1" }, { "id": "9099ed1b76f48d198d96", "model": "codegemma-7b", "organization": "", "configuration": "Completion", "score": 0.445, "raw_score": 44.5, "raw_scale": 100, "source_record": "codegemma-7b" }, { "id": "9bdec0553f7b091d4fa6", "model": "SOLAR-10.7B-Instruct-v1.0", "organization": "", "configuration": "Chat prompted", "score": 0.433, "raw_score": 43.3, "raw_scale": 100, "source_record": "SOLAR-10.7B-Instruct-v1.0" }, { "id": "d0ca7fe2486da04e757a", "model": "CodeLlama-13B", "organization": "", "configuration": "Completion", "score": 0.42700000000000005, "raw_score": 42.7, "raw_scale": 100, "source_record": "CodeLlama-13B" }, { "id": "9f677752c671779b5a34", "model": "gemma-1.1-7b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.42700000000000005, "raw_score": 42.7, "raw_scale": 100, "source_record": "gemma-1.1-7b-it" }, { "id": "c55a8bf0730a3573e29d", "model": "Mistral-7B-Instruct-v0.2", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "Mistral-7B-Instruct-v0.2" }, { "id": "005d9daebc51852f70d7", "model": "xDAN-L1-Chat-RL-v1-7B", "organization": "", "configuration": "Chat prompted", "score": 0.402, "raw_score": 40.2, "raw_scale": 100, "source_record": "xDAN-L1-Chat-RL-v1-7B" }, { "id": "de56548e9c08c87bf85f", "model": "CodeLlama-7B", "organization": "", "configuration": "Completion", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "CodeLlama-7B" }, { "id": "c5b8c76a405f96925998", "model": "StarCoder2-7B", "organization": "", "configuration": "Completion", "score": 0.354, "raw_score": 35.4, "raw_scale": 100, "source_record": "StarCoder2-7B" }, { "id": "7e284063ce25181370e5", "model": "gemma-7b", "organization": "Google", "configuration": "Completion", "score": 0.354, "raw_score": 35.4, "raw_scale": 100, "source_record": "gemma-7b" }, { "id": "92d17f2298dadfde7992", "model": "StarCoder-15B", "organization": "", "configuration": "Completion", "score": 0.341, "raw_score": 34.1, "raw_scale": 100, "source_record": "StarCoder-15B" }, { "id": "ddc6c6816a66a05bfc0d", "model": "Llama3-8B-base", "organization": "Meta", "configuration": "Completion", "score": 0.335, "raw_score": 33.5, "raw_scale": 100, "source_record": "Llama3-8B-base" }, { "id": "8c896809b7bcc37264e1", "model": "CodeGen-16B", "organization": "", "configuration": "Completion", "score": 0.32899999999999996, "raw_score": 32.9, "raw_scale": 100, "source_record": "CodeGen-16B" }, { "id": "54cc62ffa36882483ccb", "model": "Python-Code-13B", "organization": "", "configuration": "Chat prompted", "score": 0.32899999999999996, "raw_score": 32.9, "raw_scale": 100, "source_record": "Python-Code-13B" }, { "id": "bf46a364dbe17c332500", "model": "CodeT5+-16B", "organization": "", "configuration": "Completion", "score": 0.317, "raw_score": 31.7, "raw_scale": 100, "source_record": "CodeT5+-16B" }, { "id": "4df7648b2a7ce499a014", "model": "StarCoder2-3B", "organization": "", "configuration": "Completion", "score": 0.317, "raw_score": 31.7, "raw_scale": 100, "source_record": "StarCoder2-3B" }, { "id": "e9ac30646c6ce21c3c49", "model": "Zephyr β-7B", "organization": "", "configuration": "Completion", "score": 0.3, "raw_score": 30.0, "raw_scale": 100, "source_record": "Zephyr β-7B" }, { "id": "718f3dda3ad78fdcf468", "model": "CodeGen-6B", "organization": "", "configuration": "Completion", "score": 0.293, "raw_score": 29.3, "raw_scale": 100, "source_record": "CodeGen-6B" }, { "id": "79d2b13a95371d76076e", "model": "CodeT5+-6B", "organization": "", "configuration": "Completion", "score": 0.293, "raw_score": 29.3, "raw_scale": 100, "source_record": "CodeT5+-6B" }, { "id": "1d3131d05209389bab10", "model": "stable-code-3B", "organization": "", "configuration": "Completion", "score": 0.293, "raw_score": 29.3, "raw_scale": 100, "source_record": "stable-code-3B" }, { "id": "7d15a83dbab9f492b074", "model": "DeepSeek-Coder-1.3B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.287, "raw_score": 28.7, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-base" }, { "id": "d935dcc5032c427f693f", "model": "Mistral-7B", "organization": "Mistral AI", "configuration": "Completion", "score": 0.287, "raw_score": 28.7, "raw_scale": 100, "source_record": "Mistral-7B" }, { "id": "0b0cbd2addba246116e2", "model": "gemma-7b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.287, "raw_score": 28.7, "raw_scale": 100, "source_record": "gemma-7b-it" }, { "id": "3cdbec7950e8d6915ebb", "model": "codegemma-2b", "organization": "", "configuration": "Completion", "score": 0.268, "raw_score": 26.8, "raw_scale": 100, "source_record": "codegemma-2b" }, { "id": "cd10e8545ec387717186", "model": "CodeT5+-2B", "organization": "", "configuration": "Completion", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "CodeT5+-2B" }, { "id": "f5a5c93b99f6856d04db", "model": "gemma-2b", "organization": "Google", "configuration": "Completion", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "gemma-2b" }, { "id": "acb1393ea8fd0796bbdd", "model": "CodeGen-2B", "organization": "", "configuration": "Completion", "score": 0.244, "raw_score": 24.4, "raw_scale": 100, "source_record": "CodeGen-2B" }, { "id": "e51adcaf60ac5abd66c0", "model": "StarCoderBase-7B", "organization": "", "configuration": "Completion", "score": 0.244, "raw_score": 24.4, "raw_scale": 100, "source_record": "StarCoderBase-7B" }, { "id": "ed86c1d572c3c7d20e21", "model": "gemma-1.1-2b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.226, "raw_score": 22.6, "raw_scale": 100, "source_record": "gemma-1.1-2b-it" }, { "id": "020d79811354ea6b09d1", "model": "CodeGen2-16B", "organization": "", "configuration": "Completion", "score": 0.195, "raw_score": 19.5, "raw_scale": 100, "source_record": "CodeGen2-16B" }, { "id": "f5f1b422e957062346bd", "model": "CodeGen2-7B", "organization": "", "configuration": "Completion", "score": 0.183, "raw_score": 18.3, "raw_scale": 100, "source_record": "CodeGen2-7B" }, { "id": "3d508fa577945b67f890", "model": "StarCoderBase-3B", "organization": "", "configuration": "Completion", "score": 0.177, "raw_score": 17.7, "raw_scale": 100, "source_record": "StarCoderBase-3B" }, { "id": "d6404f0e75646f0178eb", "model": "gemma-2b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.177, "raw_score": 17.7, "raw_scale": 100, "source_record": "gemma-2b-it" }, { "id": "a4b98ffae08cd373c0c3", "model": "Vicuna-13B", "organization": "", "configuration": "Completion", "score": 0.171, "raw_score": 17.1, "raw_scale": 100, "source_record": "Vicuna-13B" }, { "id": "a513f9e5ac0af04017b4", "model": "CodeGen2-3B", "organization": "", "configuration": "Completion", "score": 0.159, "raw_score": 15.9, "raw_scale": 100, "source_record": "CodeGen2-3B" }, { "id": "f8b8fa3a2dc3ae0f4972", "model": "InCoder-6.7B", "organization": "", "configuration": "Completion", "score": 0.159, "raw_score": 15.9, "raw_scale": 100, "source_record": "InCoder-6.7B" }, { "id": "1ab989415086669b50fa", "model": "SantaCoder-1.1B", "organization": "", "configuration": "Completion", "score": 0.146, "raw_score": 14.6, "raw_scale": 100, "source_record": "SantaCoder-1.1B" }, { "id": "8ab35bd31c6ba5a52cb6", "model": "StarCoderBase-1B", "organization": "", "configuration": "Completion", "score": 0.146, "raw_score": 14.6, "raw_scale": 100, "source_record": "StarCoderBase-1B" }, { "id": "2c98388c12d6acb915c8", "model": "GPT-J-6B", "organization": "OpenAI", "configuration": "Completion", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "GPT-J-6B" }, { "id": "e4c0df52fc3a6d741bcc", "model": "InCoder-1.3B", "organization": "", "configuration": "Completion", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "InCoder-1.3B" }, { "id": "69405ccccd7e4e6fffaf", "model": "Vicuna-7B", "organization": "", "configuration": "Completion", "score": 0.11599999999999999, "raw_score": 11.6, "raw_scale": 100, "source_record": "Vicuna-7B" }, { "id": "e41a4b3255309240f303", "model": "CodeGen2-1B", "organization": "", "configuration": "Completion", "score": 0.11, "raw_score": 11.0, "raw_scale": 100, "source_record": "CodeGen2-1B" }, { "id": "22e1deab7fb7e75cc019", "model": "GPT-Neo-2.7B", "organization": "OpenAI", "configuration": "Completion", "score": 0.079, "raw_score": 7.9, "raw_scale": 100, "source_record": "GPT-Neo-2.7B" }, { "id": "34d38ed80133bf025f96", "model": "PolyCoder-2.7B", "organization": "", "configuration": "Completion", "score": 0.061, "raw_score": 6.1, "raw_scale": 100, "source_record": "PolyCoder-2.7B" }, { "id": "dbf8d985e3f8db3ae437", "model": "StableLM-7B", "organization": "", "configuration": "Completion", "score": 0.024, "raw_score": 2.4, "raw_scale": 100, "source_record": "StableLM-7B" }, { "id": "a16e817b9f52813e5991", "model": "zyte-1B", "organization": "", "configuration": "Chat prompted", "score": 0.024, "raw_score": 2.4, "raw_scale": 100, "source_record": "zyte-1B" } ] }, { "id": "humaneval-plus", "name": "HumanEval+", "group": "编程", "unit": "%", "metric": "pass@1", "protocol": { "en": "EvalPlus published pass@1. HumanEval has 164 tasks; MBPP uses the 399-task EvalPlus subset. Base tests and expanded + tests remain separate.", "zh": "EvalPlus 发布的 pass@1。HumanEval 为 164 道任务；MBPP 使用 EvalPlus 的 399 道题子集。原始测试与扩展 + 测试分别展示。" }, "source": { "data_url": "https://evalplus.github.io/results.json", "sha256": "e6837dc3b94f32cb1da0ced012e52edc4a722007b684df5b63da029bbd3b46d2", "retrieved_at": "2026-11-01T16:39:18.103888+00:00", "name": "EvalPlus", "url": "https://evalplus.github.io/leaderboard.html", "selector": "pass@1.humaneval+", "status": "current_snapshot", "kind": "evaluator" }, "results": [ { "id": "2a45d8a080de4798ae96", "model": "O1 Mini (Sept 2024)", "organization": "", "configuration": "Chat prompted", "score": 0.89, "raw_score": 89.0, "raw_scale": 100, "source_record": "O1 Mini (Sept 2024)" }, { "id": "084039330e5ae9df8fdd", "model": "O1 Preview (Sept 2024)", "organization": "", "configuration": "Chat prompted", "score": 0.89, "raw_score": 89.0, "raw_scale": 100, "source_record": "O1 Preview (Sept 2024)" }, { "id": "4ea990b3089f21fc6dae", "model": "GPT 4o (Aug 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.872, "raw_score": 87.2, "raw_scale": 100, "source_record": "GPT 4o (Aug 2024)" }, { "id": "0a1f27c38a7a1134eb67", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Chat prompted", "score": 0.872, "raw_score": 87.2, "raw_scale": 100, "source_record": "Qwen2.5-Coder-32B-Instruct" }, { "id": "53f47496f476362a7ce1", "model": "DeepSeek-V3 (Nov 2024)", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.866, "raw_score": 86.6, "raw_scale": 100, "source_record": "DeepSeek-V3 (Nov 2024)" }, { "id": "b392daa9ccf08c8e805c", "model": "GPT-4-Turbo (April 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.866, "raw_score": 86.6, "raw_scale": 100, "source_record": "GPT-4-Turbo (April 2024)" }, { "id": "b8c790bc669412a2379b", "model": "DeepSeek-V2.5 (Nov 2024)", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.835, "raw_score": 83.5, "raw_scale": 100, "source_record": "DeepSeek-V2.5 (Nov 2024)" }, { "id": "db1cab9d6afa1cf84290", "model": "GPT 4o Mini (July 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.835, "raw_score": 83.5, "raw_scale": 100, "source_record": "GPT 4o Mini (July 2024)" }, { "id": "531cb4f68416b94fca45", "model": "DeepSeek-Coder-V2-Instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.823, "raw_score": 82.3, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct" }, { "id": "1c46e296b7812dc418c8", "model": "Claude Sonnet 3.5 (June 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.8170000000000001, "raw_score": 81.7, "raw_scale": 100, "source_record": "Claude Sonnet 3.5 (June 2024)" }, { "id": "80d858ccbb48ab9ec53b", "model": "GPT-4-Turbo (Nov 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.8170000000000001, "raw_score": 81.7, "raw_scale": 100, "source_record": "GPT-4-Turbo (Nov 2023)" }, { "id": "769fede65d4cf9903d11", "model": "Grok Beta", "organization": "xAI", "configuration": "Chat prompted", "score": 0.805, "raw_score": 80.5, "raw_scale": 100, "source_record": "Grok Beta" }, { "id": "a139c9468fc1e37ea9ed", "model": "GPT-4 (May 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.7929999999999999, "raw_score": 79.3, "raw_scale": 100, "source_record": "GPT-4 (May 2023)" }, { "id": "0b7407a75379444a79fb", "model": "Gemini 1.5 Pro 002", "organization": "Google", "configuration": "Chat prompted", "score": 0.7929999999999999, "raw_score": 79.3, "raw_scale": 100, "source_record": "Gemini 1.5 Pro 002" }, { "id": "db1c1e11f754025a1f92", "model": "CodeQwen1.5-7B-Chat", "organization": "", "configuration": "Chat prompted", "score": 0.787, "raw_score": 78.7, "raw_scale": 100, "source_record": "CodeQwen1.5-7B-Chat" }, { "id": "691f84511d0ec15341c5", "model": "OpenCoder-8B-Instruct", "organization": "", "configuration": "Chat prompted", "score": 0.774, "raw_score": 77.4, "raw_scale": 100, "source_record": "OpenCoder-8B-Instruct" }, { "id": "62e201fb8adbc72d49fa", "model": "claude-3-opus (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.774, "raw_score": 77.4, "raw_scale": 100, "source_record": "claude-3-opus (Mar 2024)" }, { "id": "0fe26a52dd1b57b26968", "model": "Gemini 1.5 Flash 002", "organization": "Google", "configuration": "Chat prompted", "score": 0.7559999999999999, "raw_score": 75.6, "raw_scale": 100, "source_record": "Gemini 1.5 Flash 002" }, { "id": "90184c89f29471ceb2e3", "model": "DeepSeek-Coder-33B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-instruct" }, { "id": "1a2aca7e452afb3aeff3", "model": "Codestral-22B-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.738, "raw_score": 73.8, "raw_scale": 100, "source_record": "Codestral-22B-v0.1" }, { "id": "56928ca9ba87a6db3cc6", "model": "OpenCodeInterpreter-DS-33B", "organization": "", "configuration": "Chat prompted", "score": 0.738, "raw_score": 73.8, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-33B" }, { "id": "8b4c4e5236cfdb78aec9", "model": "WizardCoder-33B-V1.1", "organization": "", "configuration": "Chat prompted", "score": 0.732, "raw_score": 73.2, "raw_scale": 100, "source_record": "WizardCoder-33B-V1.1" }, { "id": "39651f4435d5050c9212", "model": "Artigenz-Coder-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.726, "raw_score": 72.6, "raw_scale": 100, "source_record": "Artigenz-Coder-DS-6.7B" }, { "id": "12cb129efc0d3d8dcb57", "model": "Llama3-70B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "Llama3-70B-instruct" }, { "id": "f7e707367b35c8316fa3", "model": "Mixtral-8x22B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct-v0.1" }, { "id": "fb1399a69666a126eaff", "model": "OpenCodeInterpreter-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-6.7B" }, { "id": "280581193c6a0c9a67dc", "model": "speechless-codellama-34B-v2.0", "organization": "", "configuration": "Chat prompted", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "speechless-codellama-34B-v2.0" }, { "id": "93181f1d7ffc42d0ed49", "model": "DeepSeek-Coder-6.7B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.713, "raw_score": 71.3, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-instruct" }, { "id": "eb111531fd091ddc90ae", "model": "DeepSeek-Coder-7B-instruct-v1.5", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.713, "raw_score": 71.3, "raw_scale": 100, "source_record": "DeepSeek-Coder-7B-instruct-v1.5" }, { "id": "d1ef871a823b03b69612", "model": "Magicoder-S-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.713, "raw_score": 71.3, "raw_scale": 100, "source_record": "Magicoder-S-DS-6.7B" }, { "id": "84cf4ef9b52e43610ec4", "model": "starchat2-15b-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.713, "raw_score": 71.3, "raw_scale": 100, "source_record": "starchat2-15b-v0.1" }, { "id": "44d35736cd73f5e16752", "model": "GPT-3.5-Turbo (Nov 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.7070000000000001, "raw_score": 70.7, "raw_scale": 100, "source_record": "GPT-3.5-Turbo (Nov 2023)" }, { "id": "1d58398274f62163ce53", "model": "code-millenials-34B", "organization": "", "configuration": "Chat prompted", "score": 0.7070000000000001, "raw_score": 70.7, "raw_scale": 100, "source_record": "code-millenials-34B" }, { "id": "d1a87920d704d9eb6a2f", "model": "databricks/dbrx-instruct", "organization": "", "configuration": "Chat prompted", "score": 0.701, "raw_score": 70.1, "raw_scale": 100, "source_record": "databricks/dbrx-instruct" }, { "id": "172fc6a41d0f943afeb2", "model": "WaveCoder-Ultra-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.695, "raw_score": 69.5, "raw_scale": 100, "source_record": "WaveCoder-Ultra-6.7B" }, { "id": "566a6f6a778e843caa18", "model": "XwinCoder-34B", "organization": "", "configuration": "Chat prompted", "score": 0.695, "raw_score": 69.5, "raw_scale": 100, "source_record": "XwinCoder-34B" }, { "id": "1cfc65ad1da9ba081cb4", "model": "claude-3-haiku (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.6890000000000001, "raw_score": 68.9, "raw_scale": 100, "source_record": "claude-3-haiku (Mar 2024)" }, { "id": "2141e7a005e32dd9e409", "model": "Magicoder-S-CL-7B", "organization": "", "configuration": "Chat prompted", "score": 0.677, "raw_score": 67.7, "raw_scale": 100, "source_record": "Magicoder-S-CL-7B" }, { "id": "f5b8f9757571067b6982", "model": "OpenChat-3.5-7B-0106", "organization": "", "configuration": "Chat prompted", "score": 0.677, "raw_score": 67.7, "raw_scale": 100, "source_record": "OpenChat-3.5-7B-0106" }, { "id": "c98656b9280aa90ed5ab", "model": "Phind-CodeLlama-34B-v2", "organization": "", "configuration": "Completion", "score": 0.6709999999999999, "raw_score": 67.1, "raw_scale": 100, "source_record": "Phind-CodeLlama-34B-v2" }, { "id": "a9c0da8f899188336f08", "model": "GPT-3.5 (May 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.665, "raw_score": 66.5, "raw_scale": 100, "source_record": "GPT-3.5 (May 2023)" }, { "id": "b8f265c234cd3f272e73", "model": "CodeLlama-70B-Instruct", "organization": "", "configuration": "Chat prompted", "score": 0.659, "raw_score": 65.9, "raw_scale": 100, "source_record": "CodeLlama-70B-Instruct" }, { "id": "4052178124a22c33fce0", "model": "WhiteRabbitNeo-33B-v1", "organization": "", "configuration": "Chat prompted", "score": 0.659, "raw_score": 65.9, "raw_scale": 100, "source_record": "WhiteRabbitNeo-33B-v1" }, { "id": "3dc355840d04012b3892", "model": "speechless-coder-ds-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.659, "raw_score": 65.9, "raw_scale": 100, "source_record": "speechless-coder-ds-6.7B" }, { "id": "d5be5926ead27aa7faa4", "model": "WizardCoder-Python-34B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.6459999999999999, "raw_score": 64.6, "raw_scale": 100, "source_record": "WizardCoder-Python-34B-V1.0" }, { "id": "7d2e36d3604d93694e87", "model": "claude-3-sonnet (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.64, "raw_score": 64.0, "raw_scale": 100, "source_record": "claude-3-sonnet (Mar 2024)" }, { "id": "8d512e3e8216a32abb97", "model": "Llama3.1-8B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.628, "raw_score": 62.8, "raw_scale": 100, "source_record": "Llama3.1-8B-instruct" }, { "id": "a54728bce2fb3a3af23e", "model": "speechless-starcoder2-15b", "organization": "", "configuration": "Chat prompted", "score": 0.628, "raw_score": 62.8, "raw_scale": 100, "source_record": "speechless-starcoder2-15b" }, { "id": "f12d2bd74dd1e3d0c0e5", "model": "Mistral Large (Mar 2024)", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.622, "raw_score": 62.2, "raw_scale": 100, "source_record": "Mistral Large (Mar 2024)" }, { "id": "85a84626a2de31308989", "model": "claude-2 (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.616, "raw_score": 61.6, "raw_scale": 100, "source_record": "claude-2 (Mar 2024)" }, { "id": "f47a7de678f154245c59", "model": "Gemini Pro 1.5", "organization": "Google", "configuration": "Chat prompted", "score": 0.61, "raw_score": 61.0, "raw_scale": 100, "source_record": "Gemini Pro 1.5" }, { "id": "d405297a8cbc9798976d", "model": "DeepSeek-Coder-1.3B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.604, "raw_score": 60.4, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-instruct" }, { "id": "c8a626855515bd14b1b4", "model": "starcoder2-15b-instruct-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.604, "raw_score": 60.4, "raw_scale": 100, "source_record": "starcoder2-15b-instruct-v0.1" }, { "id": "9e6e87afd361ba906c29", "model": "Code-290k-6.7B-Instruct", "organization": "", "configuration": "Chat prompted", "score": 0.597, "raw_score": 59.7, "raw_scale": 100, "source_record": "Code-290k-6.7B-Instruct" }, { "id": "e475320e773d474a2943", "model": "Phi-3-mini-4k-instruct", "organization": "Microsoft", "configuration": "Chat prompted", "score": 0.591, "raw_score": 59.1, "raw_scale": 100, "source_record": "Phi-3-mini-4k-instruct" }, { "id": "4afdad5104fa1b40e87e", "model": "Qwen1.5-72B-Chat", "organization": "Alibaba", "configuration": "Chat prompted", "score": 0.591, "raw_score": 59.1, "raw_scale": 100, "source_record": "Qwen1.5-72B-Chat" }, { "id": "1276ae6f21eb1bd0bb29", "model": "dolphin-2.6-mixtral-8x7b", "organization": "", "configuration": "Chat prompted", "score": 0.573, "raw_score": 57.3, "raw_scale": 100, "source_record": "dolphin-2.6-mixtral-8x7b" }, { "id": "6398e7bd9d49b2534f97", "model": "Command-R+", "organization": "", "configuration": "Chat prompted", "score": 0.5670000000000001, "raw_score": 56.7, "raw_scale": 100, "source_record": "Command-R+" }, { "id": "c75e7da3d9fe19708a6f", "model": "Llama3-8B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.5670000000000001, "raw_score": 56.7, "raw_scale": 100, "source_record": "Llama3-8B-instruct" }, { "id": "7e3a60ef3ae22f35efc4", "model": "Gemini Pro 1.0", "organization": "Google", "configuration": "Chat prompted", "score": 0.555, "raw_score": 55.5, "raw_scale": 100, "source_record": "Gemini Pro 1.0" }, { "id": "53a0daaf8f346170de88", "model": "Code-13B", "organization": "", "configuration": "Chat prompted", "score": 0.524, "raw_score": 52.4, "raw_scale": 100, "source_record": "Code-13B" }, { "id": "3fdafb78c6f7d8c92289", "model": "codegemma-7b-it", "organization": "", "configuration": "Chat prompted", "score": 0.518, "raw_score": 51.8, "raw_scale": 100, "source_record": "codegemma-7b-it" }, { "id": "3f0edba6458b50bd4cc1", "model": "speechless-starcoder2-7b", "organization": "", "configuration": "Chat prompted", "score": 0.518, "raw_score": 51.8, "raw_scale": 100, "source_record": "speechless-starcoder2-7b" }, { "id": "4907f74fca56a1e14b1a", "model": "CodeLlama-70B", "organization": "", "configuration": "Completion", "score": 0.506, "raw_score": 50.6, "raw_scale": 100, "source_record": "CodeLlama-70B" }, { "id": "4e2a62e8daafafba2736", "model": "WizardCoder-15B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.506, "raw_score": 50.6, "raw_scale": 100, "source_record": "WizardCoder-15B-V1.0" }, { "id": "82367639fb7a6486bcf4", "model": "claude-instant-1 (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.506, "raw_score": 50.6, "raw_scale": 100, "source_record": "claude-instant-1 (Mar 2024)" }, { "id": "06d2430a53a2a51bb3b0", "model": "speechless-coding-7B-16k-tora", "organization": "", "configuration": "Chat prompted", "score": 0.506, "raw_score": 50.6, "raw_scale": 100, "source_record": "speechless-coding-7B-16k-tora" }, { "id": "baa147a39d546544863b", "model": "Code-33B", "organization": "", "configuration": "Chat prompted", "score": 0.494, "raw_score": 49.4, "raw_scale": 100, "source_record": "Code-33B" }, { "id": "d2f6debe8f038a5a758b", "model": "OpenHermes-2.5-Code-290k-13B", "organization": "", "configuration": "Chat prompted", "score": 0.488, "raw_score": 48.8, "raw_scale": 100, "source_record": "OpenHermes-2.5-Code-290k-13B" }, { "id": "bb051df20f0935ee2e27", "model": "CodeQwen1.5-7B", "organization": "", "configuration": "Completion", "score": 0.457, "raw_score": 45.7, "raw_scale": 100, "source_record": "CodeQwen1.5-7B" }, { "id": "674a9924e98b9b389a8d", "model": "WizardCoder-Python-7B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.451, "raw_score": 45.1, "raw_scale": 100, "source_record": "WizardCoder-Python-7B-V1.0" }, { "id": "2799abfaf8ad6d2a2646", "model": "phi-2-2.7B", "organization": "Microsoft", "configuration": "Completion", "score": 0.451, "raw_score": 45.1, "raw_scale": 100, "source_record": "phi-2-2.7B" }, { "id": "ce783e5b1dae0a26079d", "model": "DeepSeek-Coder-33B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.445, "raw_score": 44.5, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-base" }, { "id": "273c3cc8d56437443071", "model": "CodeLlama-34B", "organization": "", "configuration": "Completion", "score": 0.439, "raw_score": 43.9, "raw_scale": 100, "source_record": "CodeLlama-34B" }, { "id": "b7a8fb76da274a850160", "model": "Mistral-codealpaca-7B", "organization": "Mistral AI", "configuration": "Completion", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "Mistral-codealpaca-7B" }, { "id": "88050667dacbe8ae23a8", "model": "MistralHermes-CodePro-7B-v1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "MistralHermes-CodePro-7B-v1" }, { "id": "9099ed1b76f48d198d96", "model": "codegemma-7b", "organization": "", "configuration": "Completion", "score": 0.415, "raw_score": 41.5, "raw_scale": 100, "source_record": "codegemma-7b" }, { "id": "db6ffffa8f44dbfa1134", "model": "speechless-code-mistral-7B-v1.0", "organization": "", "configuration": "Chat prompted", "score": 0.415, "raw_score": 41.5, "raw_scale": 100, "source_record": "speechless-code-mistral-7B-v1.0" }, { "id": "9f6a1b5fbe306eeea58f", "model": "DeepSeek-Coder-6.7B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.396, "raw_score": 39.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-base" }, { "id": "0a800dac99fb938f31f2", "model": "Mixtral-8x7B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.396, "raw_score": 39.6, "raw_scale": 100, "source_record": "Mixtral-8x7B-Instruct-v0.1" }, { "id": "d0ca7fe2486da04e757a", "model": "CodeLlama-13B", "organization": "", "configuration": "Completion", "score": 0.384, "raw_score": 38.4, "raw_scale": 100, "source_record": "CodeLlama-13B" }, { "id": "f2b2f5ff73e7fc7a96c4", "model": "StarCoder2-15B", "organization": "", "configuration": "Completion", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "StarCoder2-15B" }, { "id": "9bdec0553f7b091d4fa6", "model": "SOLAR-10.7B-Instruct-v1.0", "organization": "", "configuration": "Chat prompted", "score": 0.37200000000000005, "raw_score": 37.2, "raw_scale": 100, "source_record": "SOLAR-10.7B-Instruct-v1.0" }, { "id": "c55a8bf0730a3573e29d", "model": "Mistral-7B-Instruct-v0.2", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.36, "raw_score": 36.0, "raw_scale": 100, "source_record": "Mistral-7B-Instruct-v0.2" }, { "id": "de56548e9c08c87bf85f", "model": "CodeLlama-7B", "organization": "", "configuration": "Completion", "score": 0.354, "raw_score": 35.4, "raw_scale": 100, "source_record": "CodeLlama-7B" }, { "id": "9f677752c671779b5a34", "model": "gemma-1.1-7b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.354, "raw_score": 35.4, "raw_scale": 100, "source_record": "gemma-1.1-7b-it" }, { "id": "005d9daebc51852f70d7", "model": "xDAN-L1-Chat-RL-v1-7B", "organization": "", "configuration": "Chat prompted", "score": 0.32899999999999996, "raw_score": 32.9, "raw_scale": 100, "source_record": "xDAN-L1-Chat-RL-v1-7B" }, { "id": "54cc62ffa36882483ccb", "model": "Python-Code-13B", "organization": "", "configuration": "Chat prompted", "score": 0.305, "raw_score": 30.5, "raw_scale": 100, "source_record": "Python-Code-13B" }, { "id": "c5b8c76a405f96925998", "model": "StarCoder2-7B", "organization": "", "configuration": "Completion", "score": 0.299, "raw_score": 29.9, "raw_scale": 100, "source_record": "StarCoder2-7B" }, { "id": "ddc6c6816a66a05bfc0d", "model": "Llama3-8B-base", "organization": "Meta", "configuration": "Completion", "score": 0.293, "raw_score": 29.3, "raw_scale": 100, "source_record": "Llama3-8B-base" }, { "id": "92d17f2298dadfde7992", "model": "StarCoder-15B", "organization": "", "configuration": "Completion", "score": 0.293, "raw_score": 29.3, "raw_scale": 100, "source_record": "StarCoder-15B" }, { "id": "7e284063ce25181370e5", "model": "gemma-7b", "organization": "Google", "configuration": "Completion", "score": 0.287, "raw_score": 28.7, "raw_scale": 100, "source_record": "gemma-7b" }, { "id": "8c896809b7bcc37264e1", "model": "CodeGen-16B", "organization": "", "configuration": "Completion", "score": 0.28, "raw_score": 28.0, "raw_scale": 100, "source_record": "CodeGen-16B" }, { "id": "4df7648b2a7ce499a014", "model": "StarCoder2-3B", "organization": "", "configuration": "Completion", "score": 0.27399999999999997, "raw_score": 27.4, "raw_scale": 100, "source_record": "StarCoder2-3B" }, { "id": "bf46a364dbe17c332500", "model": "CodeT5+-16B", "organization": "", "configuration": "Completion", "score": 0.268, "raw_score": 26.8, "raw_scale": 100, "source_record": "CodeT5+-16B" }, { "id": "718f3dda3ad78fdcf468", "model": "CodeGen-6B", "organization": "", "configuration": "Completion", "score": 0.256, "raw_score": 25.6, "raw_scale": 100, "source_record": "CodeGen-6B" }, { "id": "7d15a83dbab9f492b074", "model": "DeepSeek-Coder-1.3B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.256, "raw_score": 25.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-base" }, { "id": "1d3131d05209389bab10", "model": "stable-code-3B", "organization": "", "configuration": "Completion", "score": 0.256, "raw_score": 25.6, "raw_scale": 100, "source_record": "stable-code-3B" }, { "id": "0b0cbd2addba246116e2", "model": "gemma-7b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "gemma-7b-it" }, { "id": "79d2b13a95371d76076e", "model": "CodeT5+-6B", "organization": "", "configuration": "Completion", "score": 0.244, "raw_score": 24.4, "raw_scale": 100, "source_record": "CodeT5+-6B" }, { "id": "d935dcc5032c427f693f", "model": "Mistral-7B", "organization": "Mistral AI", "configuration": "Completion", "score": 0.23800000000000002, "raw_score": 23.8, "raw_scale": 100, "source_record": "Mistral-7B" }, { "id": "e9ac30646c6ce21c3c49", "model": "Zephyr β-7B", "organization": "", "configuration": "Completion", "score": 0.23199999999999998, "raw_score": 23.2, "raw_scale": 100, "source_record": "Zephyr β-7B" }, { "id": "acb1393ea8fd0796bbdd", "model": "CodeGen-2B", "organization": "", "configuration": "Completion", "score": 0.226, "raw_score": 22.6, "raw_scale": 100, "source_record": "CodeGen-2B" }, { "id": "cd10e8545ec387717186", "model": "CodeT5+-2B", "organization": "", "configuration": "Completion", "score": 0.22, "raw_score": 22.0, "raw_scale": 100, "source_record": "CodeT5+-2B" }, { "id": "e51adcaf60ac5abd66c0", "model": "StarCoderBase-7B", "organization": "", "configuration": "Completion", "score": 0.213, "raw_score": 21.3, "raw_scale": 100, "source_record": "StarCoderBase-7B" }, { "id": "3cdbec7950e8d6915ebb", "model": "codegemma-2b", "organization": "", "configuration": "Completion", "score": 0.207, "raw_score": 20.7, "raw_scale": 100, "source_record": "codegemma-2b" }, { "id": "f5a5c93b99f6856d04db", "model": "gemma-2b", "organization": "Google", "configuration": "Completion", "score": 0.207, "raw_score": 20.7, "raw_scale": 100, "source_record": "gemma-2b" }, { "id": "f5f1b422e957062346bd", "model": "CodeGen2-7B", "organization": "", "configuration": "Completion", "score": 0.177, "raw_score": 17.7, "raw_scale": 100, "source_record": "CodeGen2-7B" }, { "id": "ed86c1d572c3c7d20e21", "model": "gemma-1.1-2b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.177, "raw_score": 17.7, "raw_scale": 100, "source_record": "gemma-1.1-2b-it" }, { "id": "020d79811354ea6b09d1", "model": "CodeGen2-16B", "organization": "", "configuration": "Completion", "score": 0.165, "raw_score": 16.5, "raw_scale": 100, "source_record": "CodeGen2-16B" }, { "id": "3d508fa577945b67f890", "model": "StarCoderBase-3B", "organization": "", "configuration": "Completion", "score": 0.159, "raw_score": 15.9, "raw_scale": 100, "source_record": "StarCoderBase-3B" }, { "id": "a4b98ffae08cd373c0c3", "model": "Vicuna-13B", "organization": "", "configuration": "Completion", "score": 0.159, "raw_score": 15.9, "raw_scale": 100, "source_record": "Vicuna-13B" }, { "id": "d6404f0e75646f0178eb", "model": "gemma-2b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.152, "raw_score": 15.2, "raw_scale": 100, "source_record": "gemma-2b-it" }, { "id": "1ab989415086669b50fa", "model": "SantaCoder-1.1B", "organization": "", "configuration": "Completion", "score": 0.14, "raw_score": 14.0, "raw_scale": 100, "source_record": "SantaCoder-1.1B" }, { "id": "a513f9e5ac0af04017b4", "model": "CodeGen2-3B", "organization": "", "configuration": "Completion", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "CodeGen2-3B" }, { "id": "f8b8fa3a2dc3ae0f4972", "model": "InCoder-6.7B", "organization": "", "configuration": "Completion", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "InCoder-6.7B" }, { "id": "8ab35bd31c6ba5a52cb6", "model": "StarCoderBase-1B", "organization": "", "configuration": "Completion", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "StarCoderBase-1B" }, { "id": "69405ccccd7e4e6fffaf", "model": "Vicuna-7B", "organization": "", "configuration": "Completion", "score": 0.11599999999999999, "raw_score": 11.6, "raw_scale": 100, "source_record": "Vicuna-7B" }, { "id": "2c98388c12d6acb915c8", "model": "GPT-J-6B", "organization": "OpenAI", "configuration": "Completion", "score": 0.11, "raw_score": 11.0, "raw_scale": 100, "source_record": "GPT-J-6B" }, { "id": "e4c0df52fc3a6d741bcc", "model": "InCoder-1.3B", "organization": "", "configuration": "Completion", "score": 0.11, "raw_score": 11.0, "raw_scale": 100, "source_record": "InCoder-1.3B" }, { "id": "e41a4b3255309240f303", "model": "CodeGen2-1B", "organization": "", "configuration": "Completion", "score": 0.091, "raw_score": 9.1, "raw_scale": 100, "source_record": "CodeGen2-1B" }, { "id": "22e1deab7fb7e75cc019", "model": "GPT-Neo-2.7B", "organization": "OpenAI", "configuration": "Completion", "score": 0.067, "raw_score": 6.7, "raw_scale": 100, "source_record": "GPT-Neo-2.7B" }, { "id": "34d38ed80133bf025f96", "model": "PolyCoder-2.7B", "organization": "", "configuration": "Completion", "score": 0.061, "raw_score": 6.1, "raw_scale": 100, "source_record": "PolyCoder-2.7B" }, { "id": "dbf8d985e3f8db3ae437", "model": "StableLM-7B", "organization": "", "configuration": "Completion", "score": 0.024, "raw_score": 2.4, "raw_scale": 100, "source_record": "StableLM-7B" }, { "id": "a16e817b9f52813e5991", "model": "zyte-1B", "organization": "", "configuration": "Chat prompted", "score": 0.018000000000000002, "raw_score": 1.8, "raw_scale": 100, "source_record": "zyte-1B" } ] }, { "id": "mbpp-evalplus", "name": "MBPP (EvalPlus subset)", "group": "编程", "unit": "%", "metric": "pass@1", "protocol": { "en": "EvalPlus published pass@1. HumanEval has 164 tasks; MBPP uses the 399-task EvalPlus subset. Base tests and expanded + tests remain separate.", "zh": "EvalPlus 发布的 pass@1。HumanEval 为 164 道任务；MBPP 使用 EvalPlus 的 399 道题子集。原始测试与扩展 + 测试分别展示。" }, "source": { "data_url": "https://evalplus.github.io/results.json", "sha256": "e6837dc3b94f32cb1da0ced012e52edc4a722007b684df5b63da029bbd3b46d2", "retrieved_at": "2026-11-01T16:39:18.103888+00:00", "name": "EvalPlus", "url": "https://evalplus.github.io/leaderboard.html", "selector": "pass@1.mbpp", "status": "current_snapshot", "kind": "evaluator" }, "results": [ { "id": "084039330e5ae9df8fdd", "model": "O1 Preview (Sept 2024)", "organization": "", "configuration": "Chat prompted", "score": 0.955, "raw_score": 95.5, "raw_scale": 100, "source_record": "O1 Preview (Sept 2024)" }, { "id": "2a45d8a080de4798ae96", "model": "O1 Mini (Sept 2024)", "organization": "", "configuration": "Chat prompted", "score": 0.9309999999999999, "raw_score": 93.1, "raw_scale": 100, "source_record": "O1 Mini (Sept 2024)" }, { "id": "0a1f27c38a7a1134eb67", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Chat prompted", "score": 0.905, "raw_score": 90.5, "raw_scale": 100, "source_record": "Qwen2.5-Coder-32B-Instruct" }, { "id": "0b7407a75379444a79fb", "model": "Gemini 1.5 Pro 002", "organization": "Google", "configuration": "Chat prompted", "score": 0.897, "raw_score": 89.7, "raw_scale": 100, "source_record": "Gemini 1.5 Pro 002" }, { "id": "1c46e296b7812dc418c8", "model": "Claude Sonnet 3.5 (June 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.894, "raw_score": 89.4, "raw_scale": 100, "source_record": "Claude Sonnet 3.5 (June 2024)" }, { "id": "531cb4f68416b94fca45", "model": "DeepSeek-Coder-V2-Instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.894, "raw_score": 89.4, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct" }, { "id": "62e201fb8adbc72d49fa", "model": "claude-3-opus (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.894, "raw_score": 89.4, "raw_scale": 100, "source_record": "claude-3-opus (Mar 2024)" }, { "id": "b8c790bc669412a2379b", "model": "DeepSeek-V2.5 (Nov 2024)", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.8759999999999999, "raw_score": 87.6, "raw_scale": 100, "source_record": "DeepSeek-V2.5 (Nov 2024)" }, { "id": "53f47496f476362a7ce1", "model": "DeepSeek-V3 (Nov 2024)", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.8759999999999999, "raw_score": 87.6, "raw_scale": 100, "source_record": "DeepSeek-V3 (Nov 2024)" }, { "id": "4ea990b3089f21fc6dae", "model": "GPT 4o (Aug 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.8759999999999999, "raw_score": 87.6, "raw_scale": 100, "source_record": "GPT 4o (Aug 2024)" }, { "id": "769fede65d4cf9903d11", "model": "Grok Beta", "organization": "xAI", "configuration": "Chat prompted", "score": 0.86, "raw_score": 86.0, "raw_scale": 100, "source_record": "Grok Beta" }, { "id": "80d858ccbb48ab9ec53b", "model": "GPT-4-Turbo (Nov 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.857, "raw_score": 85.7, "raw_scale": 100, "source_record": "GPT-4-Turbo (Nov 2023)" }, { "id": "db1cab9d6afa1cf84290", "model": "GPT 4o Mini (July 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.8540000000000001, "raw_score": 85.4, "raw_scale": 100, "source_record": "GPT 4o Mini (July 2024)" }, { "id": "0fe26a52dd1b57b26968", "model": "Gemini 1.5 Flash 002", "organization": "Google", "configuration": "Chat prompted", "score": 0.847, "raw_score": 84.7, "raw_scale": 100, "source_record": "Gemini 1.5 Flash 002" }, { "id": "7d2e36d3604d93694e87", "model": "claude-3-sonnet (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.836, "raw_score": 83.6, "raw_scale": 100, "source_record": "claude-3-sonnet (Mar 2024)" }, { "id": "44d35736cd73f5e16752", "model": "GPT-3.5-Turbo (Nov 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.825, "raw_score": 82.5, "raw_scale": 100, "source_record": "GPT-3.5-Turbo (Nov 2023)" }, { "id": "12cb129efc0d3d8dcb57", "model": "Llama3-70B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.823, "raw_score": 82.3, "raw_scale": 100, "source_record": "Llama3-70B-instruct" }, { "id": "691f84511d0ec15341c5", "model": "OpenCoder-8B-Instruct", "organization": "", "configuration": "Chat prompted", "score": 0.82, "raw_score": 82.0, "raw_scale": 100, "source_record": "OpenCoder-8B-Instruct" }, { "id": "39651f4435d5050c9212", "model": "Artigenz-Coder-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.807, "raw_score": 80.7, "raw_scale": 100, "source_record": "Artigenz-Coder-DS-6.7B" }, { "id": "90184c89f29471ceb2e3", "model": "DeepSeek-Coder-33B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.804, "raw_score": 80.4, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-instruct" }, { "id": "56928ca9ba87a6db3cc6", "model": "OpenCodeInterpreter-DS-33B", "organization": "", "configuration": "Chat prompted", "score": 0.802, "raw_score": 80.2, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-33B" }, { "id": "1cfc65ad1da9ba081cb4", "model": "claude-3-haiku (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.802, "raw_score": 80.2, "raw_scale": 100, "source_record": "claude-3-haiku (Mar 2024)" }, { "id": "db1c1e11f754025a1f92", "model": "CodeQwen1.5-7B-Chat", "organization": "", "configuration": "Chat prompted", "score": 0.794, "raw_score": 79.4, "raw_scale": 100, "source_record": "CodeQwen1.5-7B-Chat" }, { "id": "d1ef871a823b03b69612", "model": "Magicoder-S-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.794, "raw_score": 79.4, "raw_scale": 100, "source_record": "Magicoder-S-DS-6.7B" }, { "id": "4052178124a22c33fce0", "model": "WhiteRabbitNeo-33B-v1", "organization": "", "configuration": "Chat prompted", "score": 0.794, "raw_score": 79.4, "raw_scale": 100, "source_record": "WhiteRabbitNeo-33B-v1" }, { "id": "c8a626855515bd14b1b4", "model": "starcoder2-15b-instruct-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.78, "raw_score": 78.0, "raw_scale": 100, "source_record": "starcoder2-15b-instruct-v0.1" }, { "id": "566a6f6a778e843caa18", "model": "XwinCoder-34B", "organization": "", "configuration": "Chat prompted", "score": 0.77, "raw_score": 77.0, "raw_scale": 100, "source_record": "XwinCoder-34B" }, { "id": "fb1399a69666a126eaff", "model": "OpenCodeInterpreter-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.765, "raw_score": 76.5, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-6.7B" }, { "id": "1d58398274f62163ce53", "model": "code-millenials-34B", "organization": "", "configuration": "Chat prompted", "score": 0.762, "raw_score": 76.2, "raw_scale": 100, "source_record": "code-millenials-34B" }, { "id": "3dc355840d04012b3892", "model": "speechless-coder-ds-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.759, "raw_score": 75.9, "raw_scale": 100, "source_record": "speechless-coder-ds-6.7B" }, { "id": "7e3a60ef3ae22f35efc4", "model": "Gemini Pro 1.0", "organization": "Google", "configuration": "Chat prompted", "score": 0.754, "raw_score": 75.4, "raw_scale": 100, "source_record": "Gemini Pro 1.0" }, { "id": "eb111531fd091ddc90ae", "model": "DeepSeek-Coder-7B-instruct-v1.5", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.752, "raw_score": 75.2, "raw_scale": 100, "source_record": "DeepSeek-Coder-7B-instruct-v1.5" }, { "id": "d5be5926ead27aa7faa4", "model": "WizardCoder-Python-34B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.7509999999999999, "raw_score": 75.1, "raw_scale": 100, "source_record": "WizardCoder-Python-34B-V1.0" }, { "id": "93181f1d7ffc42d0ed49", "model": "DeepSeek-Coder-6.7B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.7490000000000001, "raw_score": 74.9, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-instruct" }, { "id": "172fc6a41d0f943afeb2", "model": "WaveCoder-Ultra-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.7490000000000001, "raw_score": 74.9, "raw_scale": 100, "source_record": "WaveCoder-Ultra-6.7B" }, { "id": "84cf4ef9b52e43610ec4", "model": "starchat2-15b-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.7490000000000001, "raw_score": 74.9, "raw_scale": 100, "source_record": "starchat2-15b-v0.1" }, { "id": "6398e7bd9d49b2534f97", "model": "Command-R+", "organization": "", "configuration": "Chat prompted", "score": 0.743, "raw_score": 74.3, "raw_scale": 100, "source_record": "Command-R+" }, { "id": "f7e707367b35c8316fa3", "model": "Mixtral-8x22B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.738, "raw_score": 73.8, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct-v0.1" }, { "id": "280581193c6a0c9a67dc", "model": "speechless-codellama-34B-v2.0", "organization": "", "configuration": "Chat prompted", "score": 0.738, "raw_score": 73.8, "raw_scale": 100, "source_record": "speechless-codellama-34B-v2.0" }, { "id": "bb051df20f0935ee2e27", "model": "CodeQwen1.5-7B", "organization": "", "configuration": "Completion", "score": 0.735, "raw_score": 73.5, "raw_scale": 100, "source_record": "CodeQwen1.5-7B" }, { "id": "a54728bce2fb3a3af23e", "model": "speechless-starcoder2-15b", "organization": "", "configuration": "Chat prompted", "score": 0.735, "raw_score": 73.5, "raw_scale": 100, "source_record": "speechless-starcoder2-15b" }, { "id": "f12d2bd74dd1e3d0c0e5", "model": "Mistral Large (Mar 2024)", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.728, "raw_score": 72.8, "raw_scale": 100, "source_record": "Mistral Large (Mar 2024)" }, { "id": "1a2aca7e452afb3aeff3", "model": "Codestral-22B-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.725, "raw_score": 72.5, "raw_scale": 100, "source_record": "Codestral-22B-v0.1" }, { "id": "4afdad5104fa1b40e87e", "model": "Qwen1.5-72B-Chat", "organization": "Alibaba", "configuration": "Chat prompted", "score": 0.725, "raw_score": 72.5, "raw_scale": 100, "source_record": "Qwen1.5-72B-Chat" }, { "id": "9f6a1b5fbe306eeea58f", "model": "DeepSeek-Coder-6.7B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-base" }, { "id": "2141e7a005e32dd9e409", "model": "Magicoder-S-CL-7B", "organization": "", "configuration": "Chat prompted", "score": 0.706, "raw_score": 70.6, "raw_scale": 100, "source_record": "Magicoder-S-CL-7B" }, { "id": "1276ae6f21eb1bd0bb29", "model": "dolphin-2.6-mixtral-8x7b", "organization": "", "configuration": "Chat prompted", "score": 0.706, "raw_score": 70.6, "raw_scale": 100, "source_record": "dolphin-2.6-mixtral-8x7b" }, { "id": "3fdafb78c6f7d8c92289", "model": "codegemma-7b-it", "organization": "", "configuration": "Chat prompted", "score": 0.7040000000000001, "raw_score": 70.4, "raw_scale": 100, "source_record": "codegemma-7b-it" }, { "id": "273c3cc8d56437443071", "model": "CodeLlama-34B", "organization": "", "configuration": "Completion", "score": 0.693, "raw_score": 69.3, "raw_scale": 100, "source_record": "CodeLlama-34B" }, { "id": "8d512e3e8216a32abb97", "model": "Llama3.1-8B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.6829999999999999, "raw_score": 68.3, "raw_scale": 100, "source_record": "Llama3.1-8B-instruct" }, { "id": "d1a87920d704d9eb6a2f", "model": "databricks/dbrx-instruct", "organization": "", "configuration": "Chat prompted", "score": 0.672, "raw_score": 67.2, "raw_scale": 100, "source_record": "databricks/dbrx-instruct" }, { "id": "3f0edba6458b50bd4cc1", "model": "speechless-starcoder2-7b", "organization": "", "configuration": "Chat prompted", "score": 0.667, "raw_score": 66.7, "raw_scale": 100, "source_record": "speechless-starcoder2-7b" }, { "id": "e475320e773d474a2943", "model": "Phi-3-mini-4k-instruct", "organization": "Microsoft", "configuration": "Chat prompted", "score": 0.659, "raw_score": 65.9, "raw_scale": 100, "source_record": "Phi-3-mini-4k-instruct" }, { "id": "d405297a8cbc9798976d", "model": "DeepSeek-Coder-1.3B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.653, "raw_score": 65.3, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-instruct" }, { "id": "9099ed1b76f48d198d96", "model": "codegemma-7b", "organization": "", "configuration": "Completion", "score": 0.6509999999999999, "raw_score": 65.1, "raw_scale": 100, "source_record": "codegemma-7b" }, { "id": "c75e7da3d9fe19708a6f", "model": "Llama3-8B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.6459999999999999, "raw_score": 64.6, "raw_scale": 100, "source_record": "Llama3-8B-instruct" }, { "id": "4e2a62e8daafafba2736", "model": "WizardCoder-15B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.643, "raw_score": 64.3, "raw_scale": 100, "source_record": "WizardCoder-15B-V1.0" }, { "id": "06d2430a53a2a51bb3b0", "model": "speechless-coding-7B-16k-tora", "organization": "", "configuration": "Chat prompted", "score": 0.642, "raw_score": 64.2, "raw_scale": 100, "source_record": "speechless-coding-7B-16k-tora" }, { "id": "2799abfaf8ad6d2a2646", "model": "phi-2-2.7B", "organization": "Microsoft", "configuration": "Completion", "score": 0.64, "raw_score": 64.0, "raw_scale": 100, "source_record": "phi-2-2.7B" }, { "id": "f5b8f9757571067b6982", "model": "OpenChat-3.5-7B-0106", "organization": "", "configuration": "Chat prompted", "score": 0.638, "raw_score": 63.8, "raw_scale": 100, "source_record": "OpenChat-3.5-7B-0106" }, { "id": "d0ca7fe2486da04e757a", "model": "CodeLlama-13B", "organization": "", "configuration": "Completion", "score": 0.635, "raw_score": 63.5, "raw_scale": 100, "source_record": "CodeLlama-13B" }, { "id": "ddc6c6816a66a05bfc0d", "model": "Llama3-8B-base", "organization": "Meta", "configuration": "Completion", "score": 0.614, "raw_score": 61.4, "raw_scale": 100, "source_record": "Llama3-8B-base" }, { "id": "de56548e9c08c87bf85f", "model": "CodeLlama-7B", "organization": "", "configuration": "Completion", "score": 0.595, "raw_score": 59.5, "raw_scale": 100, "source_record": "CodeLlama-7B" }, { "id": "0a800dac99fb938f31f2", "model": "Mixtral-8x7B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.595, "raw_score": 59.5, "raw_scale": 100, "source_record": "Mixtral-8x7B-Instruct-v0.1" }, { "id": "674a9924e98b9b389a8d", "model": "WizardCoder-Python-7B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.585, "raw_score": 58.5, "raw_scale": 100, "source_record": "WizardCoder-Python-7B-V1.0" }, { "id": "88050667dacbe8ae23a8", "model": "MistralHermes-CodePro-7B-v1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.574, "raw_score": 57.4, "raw_scale": 100, "source_record": "MistralHermes-CodePro-7B-v1" }, { "id": "db6ffffa8f44dbfa1134", "model": "speechless-code-mistral-7B-v1.0", "organization": "", "configuration": "Chat prompted", "score": 0.574, "raw_score": 57.4, "raw_scale": 100, "source_record": "speechless-code-mistral-7B-v1.0" }, { "id": "9f677752c671779b5a34", "model": "gemma-1.1-7b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.5710000000000001, "raw_score": 57.1, "raw_scale": 100, "source_record": "gemma-1.1-7b-it" }, { "id": "7d15a83dbab9f492b074", "model": "DeepSeek-Coder-1.3B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.569, "raw_score": 56.9, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-base" }, { "id": "bf46a364dbe17c332500", "model": "CodeT5+-16B", "organization": "", "configuration": "Completion", "score": 0.5660000000000001, "raw_score": 56.6, "raw_scale": 100, "source_record": "CodeT5+-16B" }, { "id": "3cdbec7950e8d6915ebb", "model": "codegemma-2b", "organization": "", "configuration": "Completion", "score": 0.556, "raw_score": 55.6, "raw_scale": 100, "source_record": "codegemma-2b" }, { "id": "92d17f2298dadfde7992", "model": "StarCoder-15B", "organization": "", "configuration": "Completion", "score": 0.551, "raw_score": 55.1, "raw_scale": 100, "source_record": "StarCoder-15B" }, { "id": "1d3131d05209389bab10", "model": "stable-code-3B", "organization": "", "configuration": "Completion", "score": 0.5479999999999999, "raw_score": 54.8, "raw_scale": 100, "source_record": "stable-code-3B" }, { "id": "8c896809b7bcc37264e1", "model": "CodeGen-16B", "organization": "", "configuration": "Completion", "score": 0.542, "raw_score": 54.2, "raw_scale": 100, "source_record": "CodeGen-16B" }, { "id": "79d2b13a95371d76076e", "model": "CodeT5+-6B", "organization": "", "configuration": "Completion", "score": 0.529, "raw_score": 52.9, "raw_scale": 100, "source_record": "CodeT5+-6B" }, { "id": "7e284063ce25181370e5", "model": "gemma-7b", "organization": "Google", "configuration": "Completion", "score": 0.526, "raw_score": 52.6, "raw_scale": 100, "source_record": "gemma-7b" }, { "id": "d2f6debe8f038a5a758b", "model": "OpenHermes-2.5-Code-290k-13B", "organization": "", "configuration": "Chat prompted", "score": 0.524, "raw_score": 52.4, "raw_scale": 100, "source_record": "OpenHermes-2.5-Code-290k-13B" }, { "id": "d935dcc5032c427f693f", "model": "Mistral-7B", "organization": "Mistral AI", "configuration": "Completion", "score": 0.519, "raw_score": 51.9, "raw_scale": 100, "source_record": "Mistral-7B" }, { "id": "718f3dda3ad78fdcf468", "model": "CodeGen-6B", "organization": "", "configuration": "Completion", "score": 0.508, "raw_score": 50.8, "raw_scale": 100, "source_record": "CodeGen-6B" }, { "id": "005d9daebc51852f70d7", "model": "xDAN-L1-Chat-RL-v1-7B", "organization": "", "configuration": "Chat prompted", "score": 0.503, "raw_score": 50.3, "raw_scale": 100, "source_record": "xDAN-L1-Chat-RL-v1-7B" }, { "id": "cd10e8545ec387717186", "model": "CodeT5+-2B", "organization": "", "configuration": "Completion", "score": 0.484, "raw_score": 48.4, "raw_scale": 100, "source_record": "CodeT5+-2B" }, { "id": "0b0cbd2addba246116e2", "model": "gemma-7b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.47100000000000003, "raw_score": 47.1, "raw_scale": 100, "source_record": "gemma-7b-it" }, { "id": "acb1393ea8fd0796bbdd", "model": "CodeGen-2B", "organization": "", "configuration": "Completion", "score": 0.46299999999999997, "raw_score": 46.3, "raw_scale": 100, "source_record": "CodeGen-2B" }, { "id": "c55a8bf0730a3573e29d", "model": "Mistral-7B-Instruct-v0.2", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.447, "raw_score": 44.7, "raw_scale": 100, "source_record": "Mistral-7B-Instruct-v0.2" }, { "id": "9bdec0553f7b091d4fa6", "model": "SOLAR-10.7B-Instruct-v1.0", "organization": "", "configuration": "Chat prompted", "score": 0.439, "raw_score": 43.9, "raw_scale": 100, "source_record": "SOLAR-10.7B-Instruct-v1.0" }, { "id": "e9ac30646c6ce21c3c49", "model": "Zephyr β-7B", "organization": "", "configuration": "Completion", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "Zephyr β-7B" }, { "id": "f5a5c93b99f6856d04db", "model": "gemma-2b", "organization": "Google", "configuration": "Completion", "score": 0.418, "raw_score": 41.8, "raw_scale": 100, "source_record": "gemma-2b" }, { "id": "ed86c1d572c3c7d20e21", "model": "gemma-1.1-2b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.298, "raw_score": 29.8, "raw_scale": 100, "source_record": "gemma-1.1-2b-it" }, { "id": "a16e817b9f52813e5991", "model": "zyte-1B", "organization": "", "configuration": "Chat prompted", "score": 0.098, "raw_score": 9.8, "raw_scale": 100, "source_record": "zyte-1B" } ] }, { "id": "mbpp-plus", "name": "MBPP+", "group": "编程", "unit": "%", "metric": "pass@1", "protocol": { "en": "EvalPlus published pass@1. HumanEval has 164 tasks; MBPP uses the 399-task EvalPlus subset. Base tests and expanded + tests remain separate.", "zh": "EvalPlus 发布的 pass@1。HumanEval 为 164 道任务；MBPP 使用 EvalPlus 的 399 道题子集。原始测试与扩展 + 测试分别展示。" }, "source": { "data_url": "https://evalplus.github.io/results.json", "sha256": "e6837dc3b94f32cb1da0ced012e52edc4a722007b684df5b63da029bbd3b46d2", "retrieved_at": "2026-11-01T16:39:18.103888+00:00", "name": "EvalPlus", "url": "https://evalplus.github.io/leaderboard.html", "selector": "pass@1.mbpp+", "status": "current_snapshot", "kind": "evaluator" }, "results": [ { "id": "084039330e5ae9df8fdd", "model": "O1 Preview (Sept 2024)", "organization": "", "configuration": "Chat prompted", "score": 0.802, "raw_score": 80.2, "raw_scale": 100, "source_record": "O1 Preview (Sept 2024)" }, { "id": "2a45d8a080de4798ae96", "model": "O1 Mini (Sept 2024)", "organization": "", "configuration": "Chat prompted", "score": 0.7879999999999999, "raw_score": 78.8, "raw_scale": 100, "source_record": "O1 Mini (Sept 2024)" }, { "id": "0a1f27c38a7a1134eb67", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Chat prompted", "score": 0.77, "raw_score": 77.0, "raw_scale": 100, "source_record": "Qwen2.5-Coder-32B-Instruct" }, { "id": "531cb4f68416b94fca45", "model": "DeepSeek-Coder-V2-Instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.7509999999999999, "raw_score": 75.1, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct" }, { "id": "0b7407a75379444a79fb", "model": "Gemini 1.5 Pro 002", "organization": "Google", "configuration": "Chat prompted", "score": 0.746, "raw_score": 74.6, "raw_scale": 100, "source_record": "Gemini 1.5 Pro 002" }, { "id": "1c46e296b7812dc418c8", "model": "Claude Sonnet 3.5 (June 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.743, "raw_score": 74.3, "raw_scale": 100, "source_record": "Claude Sonnet 3.5 (June 2024)" }, { "id": "b8c790bc669412a2379b", "model": "DeepSeek-V2.5 (Nov 2024)", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.741, "raw_score": 74.1, "raw_scale": 100, "source_record": "DeepSeek-V2.5 (Nov 2024)" }, { "id": "80d858ccbb48ab9ec53b", "model": "GPT-4-Turbo (Nov 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.733, "raw_score": 73.3, "raw_scale": 100, "source_record": "GPT-4-Turbo (Nov 2023)" }, { "id": "62e201fb8adbc72d49fa", "model": "claude-3-opus (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.733, "raw_score": 73.3, "raw_scale": 100, "source_record": "claude-3-opus (Mar 2024)" }, { "id": "53f47496f476362a7ce1", "model": "DeepSeek-V3 (Nov 2024)", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.73, "raw_score": 73.0, "raw_scale": 100, "source_record": "DeepSeek-V3 (Nov 2024)" }, { "id": "4ea990b3089f21fc6dae", "model": "GPT 4o (Aug 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.722, "raw_score": 72.2, "raw_scale": 100, "source_record": "GPT 4o (Aug 2024)" }, { "id": "db1cab9d6afa1cf84290", "model": "GPT 4o Mini (July 2024)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.722, "raw_score": 72.2, "raw_scale": 100, "source_record": "GPT 4o Mini (July 2024)" }, { "id": "691f84511d0ec15341c5", "model": "OpenCoder-8B-Instruct", "organization": "", "configuration": "Chat prompted", "score": 0.7140000000000001, "raw_score": 71.4, "raw_scale": 100, "source_record": "OpenCoder-8B-Instruct" }, { "id": "90184c89f29471ceb2e3", "model": "DeepSeek-Coder-33B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.701, "raw_score": 70.1, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-instruct" }, { "id": "44d35736cd73f5e16752", "model": "GPT-3.5-Turbo (Nov 2023)", "organization": "OpenAI", "configuration": "Chat prompted", "score": 0.6970000000000001, "raw_score": 69.7, "raw_scale": 100, "source_record": "GPT-3.5-Turbo (Nov 2023)" }, { "id": "39651f4435d5050c9212", "model": "Artigenz-Coder-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.696, "raw_score": 69.6, "raw_scale": 100, "source_record": "Artigenz-Coder-DS-6.7B" }, { "id": "7d2e36d3604d93694e87", "model": "claude-3-sonnet (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.693, "raw_score": 69.3, "raw_scale": 100, "source_record": "claude-3-sonnet (Mar 2024)" }, { "id": "db1c1e11f754025a1f92", "model": "CodeQwen1.5-7B-Chat", "organization": "", "configuration": "Chat prompted", "score": 0.69, "raw_score": 69.0, "raw_scale": 100, "source_record": "CodeQwen1.5-7B-Chat" }, { "id": "12cb129efc0d3d8dcb57", "model": "Llama3-70B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.69, "raw_score": 69.0, "raw_scale": 100, "source_record": "Llama3-70B-instruct" }, { "id": "d1ef871a823b03b69612", "model": "Magicoder-S-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.69, "raw_score": 69.0, "raw_scale": 100, "source_record": "Magicoder-S-DS-6.7B" }, { "id": "1cfc65ad1da9ba081cb4", "model": "claude-3-haiku (Mar 2024)", "organization": "Anthropic", "configuration": "Chat prompted", "score": 0.688, "raw_score": 68.8, "raw_scale": 100, "source_record": "claude-3-haiku (Mar 2024)" }, { "id": "56928ca9ba87a6db3cc6", "model": "OpenCodeInterpreter-DS-33B", "organization": "", "configuration": "Chat prompted", "score": 0.685, "raw_score": 68.5, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-33B" }, { "id": "0fe26a52dd1b57b26968", "model": "Gemini 1.5 Flash 002", "organization": "Google", "configuration": "Chat prompted", "score": 0.675, "raw_score": 67.5, "raw_scale": 100, "source_record": "Gemini 1.5 Flash 002" }, { "id": "4052178124a22c33fce0", "model": "WhiteRabbitNeo-33B-v1", "organization": "", "configuration": "Chat prompted", "score": 0.669, "raw_score": 66.9, "raw_scale": 100, "source_record": "WhiteRabbitNeo-33B-v1" }, { "id": "fb1399a69666a126eaff", "model": "OpenCodeInterpreter-DS-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.664, "raw_score": 66.4, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-6.7B" }, { "id": "93181f1d7ffc42d0ed49", "model": "DeepSeek-Coder-6.7B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.6559999999999999, "raw_score": 65.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-instruct" }, { "id": "769fede65d4cf9903d11", "model": "Grok Beta", "organization": "xAI", "configuration": "Chat prompted", "score": 0.6559999999999999, "raw_score": 65.6, "raw_scale": 100, "source_record": "Grok Beta" }, { "id": "c8a626855515bd14b1b4", "model": "starcoder2-15b-instruct-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.6509999999999999, "raw_score": 65.1, "raw_scale": 100, "source_record": "starcoder2-15b-instruct-v0.1" }, { "id": "566a6f6a778e843caa18", "model": "XwinCoder-34B", "organization": "", "configuration": "Chat prompted", "score": 0.648, "raw_score": 64.8, "raw_scale": 100, "source_record": "XwinCoder-34B" }, { "id": "1d58398274f62163ce53", "model": "code-millenials-34B", "organization": "", "configuration": "Chat prompted", "score": 0.6459999999999999, "raw_score": 64.6, "raw_scale": 100, "source_record": "code-millenials-34B" }, { "id": "84cf4ef9b52e43610ec4", "model": "starchat2-15b-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.6459999999999999, "raw_score": 64.6, "raw_scale": 100, "source_record": "starchat2-15b-v0.1" }, { "id": "3dc355840d04012b3892", "model": "speechless-coder-ds-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.644, "raw_score": 64.4, "raw_scale": 100, "source_record": "speechless-coder-ds-6.7B" }, { "id": "f7e707367b35c8316fa3", "model": "Mixtral-8x22B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.643, "raw_score": 64.3, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct-v0.1" }, { "id": "6398e7bd9d49b2534f97", "model": "Command-R+", "organization": "", "configuration": "Chat prompted", "score": 0.635, "raw_score": 63.5, "raw_scale": 100, "source_record": "Command-R+" }, { "id": "172fc6a41d0f943afeb2", "model": "WaveCoder-Ultra-6.7B", "organization": "", "configuration": "Chat prompted", "score": 0.635, "raw_score": 63.5, "raw_scale": 100, "source_record": "WaveCoder-Ultra-6.7B" }, { "id": "d5be5926ead27aa7faa4", "model": "WizardCoder-Python-34B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.632, "raw_score": 63.2, "raw_scale": 100, "source_record": "WizardCoder-Python-34B-V1.0" }, { "id": "a54728bce2fb3a3af23e", "model": "speechless-starcoder2-15b", "organization": "", "configuration": "Chat prompted", "score": 0.624, "raw_score": 62.4, "raw_scale": 100, "source_record": "speechless-starcoder2-15b" }, { "id": "eb111531fd091ddc90ae", "model": "DeepSeek-Coder-7B-instruct-v1.5", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.622, "raw_score": 62.2, "raw_scale": 100, "source_record": "DeepSeek-Coder-7B-instruct-v1.5" }, { "id": "1a2aca7e452afb3aeff3", "model": "Codestral-22B-v0.1", "organization": "", "configuration": "Chat prompted", "score": 0.619, "raw_score": 61.9, "raw_scale": 100, "source_record": "Codestral-22B-v0.1" }, { "id": "4afdad5104fa1b40e87e", "model": "Qwen1.5-72B-Chat", "organization": "Alibaba", "configuration": "Chat prompted", "score": 0.616, "raw_score": 61.6, "raw_scale": 100, "source_record": "Qwen1.5-72B-Chat" }, { "id": "7e3a60ef3ae22f35efc4", "model": "Gemini Pro 1.0", "organization": "Google", "configuration": "Chat prompted", "score": 0.614, "raw_score": 61.4, "raw_scale": 100, "source_record": "Gemini Pro 1.0" }, { "id": "280581193c6a0c9a67dc", "model": "speechless-codellama-34B-v2.0", "organization": "", "configuration": "Chat prompted", "score": 0.614, "raw_score": 61.4, "raw_scale": 100, "source_record": "speechless-codellama-34B-v2.0" }, { "id": "bb051df20f0935ee2e27", "model": "CodeQwen1.5-7B", "organization": "", "configuration": "Completion", "score": 0.608, "raw_score": 60.8, "raw_scale": 100, "source_record": "CodeQwen1.5-7B" }, { "id": "2141e7a005e32dd9e409", "model": "Magicoder-S-CL-7B", "organization": "", "configuration": "Chat prompted", "score": 0.601, "raw_score": 60.1, "raw_scale": 100, "source_record": "Magicoder-S-CL-7B" }, { "id": "f12d2bd74dd1e3d0c0e5", "model": "Mistral Large (Mar 2024)", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.595, "raw_score": 59.5, "raw_scale": 100, "source_record": "Mistral Large (Mar 2024)" }, { "id": "1276ae6f21eb1bd0bb29", "model": "dolphin-2.6-mixtral-8x7b", "organization": "", "configuration": "Chat prompted", "score": 0.59, "raw_score": 59.0, "raw_scale": 100, "source_record": "dolphin-2.6-mixtral-8x7b" }, { "id": "9f6a1b5fbe306eeea58f", "model": "DeepSeek-Coder-6.7B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.5870000000000001, "raw_score": 58.7, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-base" }, { "id": "3fdafb78c6f7d8c92289", "model": "codegemma-7b-it", "organization": "", "configuration": "Chat prompted", "score": 0.569, "raw_score": 56.9, "raw_scale": 100, "source_record": "codegemma-7b-it" }, { "id": "273c3cc8d56437443071", "model": "CodeLlama-34B", "organization": "", "configuration": "Completion", "score": 0.563, "raw_score": 56.3, "raw_scale": 100, "source_record": "CodeLlama-34B" }, { "id": "3f0edba6458b50bd4cc1", "model": "speechless-starcoder2-7b", "organization": "", "configuration": "Chat prompted", "score": 0.563, "raw_score": 56.3, "raw_scale": 100, "source_record": "speechless-starcoder2-7b" }, { "id": "d1a87920d704d9eb6a2f", "model": "databricks/dbrx-instruct", "organization": "", "configuration": "Chat prompted", "score": 0.5579999999999999, "raw_score": 55.8, "raw_scale": 100, "source_record": "databricks/dbrx-instruct" }, { "id": "8d512e3e8216a32abb97", "model": "Llama3.1-8B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.556, "raw_score": 55.6, "raw_scale": 100, "source_record": "Llama3.1-8B-instruct" }, { "id": "d405297a8cbc9798976d", "model": "DeepSeek-Coder-1.3B-instruct", "organization": "DeepSeek", "configuration": "Chat prompted", "score": 0.5479999999999999, "raw_score": 54.8, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-instruct" }, { "id": "c75e7da3d9fe19708a6f", "model": "Llama3-8B-instruct", "organization": "Meta", "configuration": "Chat prompted", "score": 0.5479999999999999, "raw_score": 54.8, "raw_scale": 100, "source_record": "Llama3-8B-instruct" }, { "id": "f5b8f9757571067b6982", "model": "OpenChat-3.5-7B-0106", "organization": "", "configuration": "Chat prompted", "score": 0.545, "raw_score": 54.5, "raw_scale": 100, "source_record": "OpenChat-3.5-7B-0106" }, { "id": "e475320e773d474a2943", "model": "Phi-3-mini-4k-instruct", "organization": "Microsoft", "configuration": "Chat prompted", "score": 0.542, "raw_score": 54.2, "raw_scale": 100, "source_record": "Phi-3-mini-4k-instruct" }, { "id": "4e2a62e8daafafba2736", "model": "WizardCoder-15B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.542, "raw_score": 54.2, "raw_scale": 100, "source_record": "WizardCoder-15B-V1.0" }, { "id": "2799abfaf8ad6d2a2646", "model": "phi-2-2.7B", "organization": "Microsoft", "configuration": "Completion", "score": 0.542, "raw_score": 54.2, "raw_scale": 100, "source_record": "phi-2-2.7B" }, { "id": "d0ca7fe2486da04e757a", "model": "CodeLlama-13B", "organization": "", "configuration": "Completion", "score": 0.526, "raw_score": 52.6, "raw_scale": 100, "source_record": "CodeLlama-13B" }, { "id": "9099ed1b76f48d198d96", "model": "codegemma-7b", "organization": "", "configuration": "Completion", "score": 0.524, "raw_score": 52.4, "raw_scale": 100, "source_record": "codegemma-7b" }, { "id": "ddc6c6816a66a05bfc0d", "model": "Llama3-8B-base", "organization": "Meta", "configuration": "Completion", "score": 0.516, "raw_score": 51.6, "raw_scale": 100, "source_record": "Llama3-8B-base" }, { "id": "06d2430a53a2a51bb3b0", "model": "speechless-coding-7B-16k-tora", "organization": "", "configuration": "Chat prompted", "score": 0.506, "raw_score": 50.6, "raw_scale": 100, "source_record": "speechless-coding-7B-16k-tora" }, { "id": "0a800dac99fb938f31f2", "model": "Mixtral-8x7B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.49700000000000005, "raw_score": 49.7, "raw_scale": 100, "source_record": "Mixtral-8x7B-Instruct-v0.1" }, { "id": "674a9924e98b9b389a8d", "model": "WizardCoder-Python-7B-V1.0", "organization": "", "configuration": "Chat prompted", "score": 0.495, "raw_score": 49.5, "raw_scale": 100, "source_record": "WizardCoder-Python-7B-V1.0" }, { "id": "db6ffffa8f44dbfa1134", "model": "speechless-code-mistral-7B-v1.0", "organization": "", "configuration": "Chat prompted", "score": 0.48700000000000004, "raw_score": 48.7, "raw_scale": 100, "source_record": "speechless-code-mistral-7B-v1.0" }, { "id": "7d15a83dbab9f492b074", "model": "DeepSeek-Coder-1.3B-base", "organization": "DeepSeek", "configuration": "Completion", "score": 0.479, "raw_score": 47.9, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-base" }, { "id": "bf46a364dbe17c332500", "model": "CodeT5+-16B", "organization": "", "configuration": "Completion", "score": 0.47100000000000003, "raw_score": 47.1, "raw_scale": 100, "source_record": "CodeT5+-16B" }, { "id": "de56548e9c08c87bf85f", "model": "CodeLlama-7B", "organization": "", "configuration": "Completion", "score": 0.46799999999999997, "raw_score": 46.8, "raw_scale": 100, "source_record": "CodeLlama-7B" }, { "id": "3cdbec7950e8d6915ebb", "model": "codegemma-2b", "organization": "", "configuration": "Completion", "score": 0.466, "raw_score": 46.6, "raw_scale": 100, "source_record": "codegemma-2b" }, { "id": "88050667dacbe8ae23a8", "model": "MistralHermes-CodePro-7B-v1", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.46399999999999997, "raw_score": 46.4, "raw_scale": 100, "source_record": "MistralHermes-CodePro-7B-v1" }, { "id": "92d17f2298dadfde7992", "model": "StarCoder-15B", "organization": "", "configuration": "Completion", "score": 0.461, "raw_score": 46.1, "raw_scale": 100, "source_record": "StarCoder-15B" }, { "id": "d2f6debe8f038a5a758b", "model": "OpenHermes-2.5-Code-290k-13B", "organization": "", "configuration": "Chat prompted", "score": 0.45799999999999996, "raw_score": 45.8, "raw_scale": 100, "source_record": "OpenHermes-2.5-Code-290k-13B" }, { "id": "1d3131d05209389bab10", "model": "stable-code-3B", "organization": "", "configuration": "Completion", "score": 0.45799999999999996, "raw_score": 45.8, "raw_scale": 100, "source_record": "stable-code-3B" }, { "id": "8c896809b7bcc37264e1", "model": "CodeGen-16B", "organization": "", "configuration": "Completion", "score": 0.455, "raw_score": 45.5, "raw_scale": 100, "source_record": "CodeGen-16B" }, { "id": "9f677752c671779b5a34", "model": "gemma-1.1-7b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.45, "raw_score": 45.0, "raw_scale": 100, "source_record": "gemma-1.1-7b-it" }, { "id": "7e284063ce25181370e5", "model": "gemma-7b", "organization": "Google", "configuration": "Completion", "score": 0.434, "raw_score": 43.4, "raw_scale": 100, "source_record": "gemma-7b" }, { "id": "718f3dda3ad78fdcf468", "model": "CodeGen-6B", "organization": "", "configuration": "Completion", "score": 0.429, "raw_score": 42.9, "raw_scale": 100, "source_record": "CodeGen-6B" }, { "id": "d935dcc5032c427f693f", "model": "Mistral-7B", "organization": "Mistral AI", "configuration": "Completion", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "Mistral-7B" }, { "id": "79d2b13a95371d76076e", "model": "CodeT5+-6B", "organization": "", "configuration": "Completion", "score": 0.415, "raw_score": 41.5, "raw_scale": 100, "source_record": "CodeT5+-6B" }, { "id": "005d9daebc51852f70d7", "model": "xDAN-L1-Chat-RL-v1-7B", "organization": "", "configuration": "Chat prompted", "score": 0.413, "raw_score": 41.3, "raw_scale": 100, "source_record": "xDAN-L1-Chat-RL-v1-7B" }, { "id": "cd10e8545ec387717186", "model": "CodeT5+-2B", "organization": "", "configuration": "Completion", "score": 0.381, "raw_score": 38.1, "raw_scale": 100, "source_record": "CodeT5+-2B" }, { "id": "c55a8bf0730a3573e29d", "model": "Mistral-7B-Instruct-v0.2", "organization": "Mistral AI", "configuration": "Chat prompted", "score": 0.37, "raw_score": 37.0, "raw_scale": 100, "source_record": "Mistral-7B-Instruct-v0.2" }, { "id": "0b0cbd2addba246116e2", "model": "gemma-7b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.368, "raw_score": 36.8, "raw_scale": 100, "source_record": "gemma-7b-it" }, { "id": "9bdec0553f7b091d4fa6", "model": "SOLAR-10.7B-Instruct-v1.0", "organization": "", "configuration": "Chat prompted", "score": 0.36200000000000004, "raw_score": 36.2, "raw_scale": 100, "source_record": "SOLAR-10.7B-Instruct-v1.0" }, { "id": "acb1393ea8fd0796bbdd", "model": "CodeGen-2B", "organization": "", "configuration": "Completion", "score": 0.36, "raw_score": 36.0, "raw_scale": 100, "source_record": "CodeGen-2B" }, { "id": "e9ac30646c6ce21c3c49", "model": "Zephyr β-7B", "organization": "", "configuration": "Completion", "score": 0.34700000000000003, "raw_score": 34.7, "raw_scale": 100, "source_record": "Zephyr β-7B" }, { "id": "f5a5c93b99f6856d04db", "model": "gemma-2b", "organization": "Google", "configuration": "Completion", "score": 0.341, "raw_score": 34.1, "raw_scale": 100, "source_record": "gemma-2b" }, { "id": "ed86c1d572c3c7d20e21", "model": "gemma-1.1-2b-it", "organization": "Google", "configuration": "Chat prompted", "score": 0.233, "raw_score": 23.3, "raw_scale": 100, "source_record": "gemma-1.1-2b-it" }, { "id": "a16e817b9f52813e5991", "model": "zyte-1B", "organization": "", "configuration": "Chat prompted", "score": 0.075, "raw_score": 7.5, "raw_scale": 100, "source_record": "zyte-1B" } ] }, { "id": "webarena", "name": "WebArena", "group": "Agent 与工具", "unit": "%", "metric": "Task success", "protocol": { "en": "Official submission leaderboard; agent scaffolds and input modes vary and are retained. Scores are reported by submitting teams.", "zh": "官方收录的提交榜单；Agent 框架与输入模式可能不同，随结果保留。成绩由提交团队报告。" }, "source": { "data_url": "https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ/export?format=csv&gid=0", "sha256": "12af5577f23efc93b2d1f9209dd924ed28db00665cae43ad6b9ad421ae91365b", "retrieved_at": "2026-11-02T05:57:39.509503+00:00", "name": "WebArena authors", "url": "https://webarena.dev/og/", "selector": "Official linked submission sheet; Success Rate(%); exclude human/random baselines", "status": "current_snapshot", "kind": "official_submissions", "artifacts": [ { "data_url": "https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ/export?format=xlsx", "sha256": "0aceb7e0a4b90bb8f2b9076c436585c2cd8bd900642765c407d1bbf55296517d", "retrieved_at": "2026-11-02T05:59:32.071345+00:00" } ] }, "results": [ { "id": "80f23670088946023b1a", "model": "Narada AI Code", "organization": "", "configuration": "Narada AI Code", "score": 0.861, "raw_score": 86.1, "raw_scale": 100, "source_record": "Narada AI Code|Narada AI Code", "source_date": "09/2026", "report_url": "https://narada.ai/blog/introducing-narada-code-operatormini" }, { "id": "4e5fb00e0132e99bf585", "model": "Deepseek v3.2", "organization": "DeepSeek", "configuration": "WebTactix", "score": 0.743, "raw_score": 74.3, "raw_scale": 100, "source_record": "Deepseek v3.2|WebTactix", "source_date": "02/2026", "report_url": "https://paper-submission-anoymous.github.io/webtactix_introduction/" }, { "id": "7e1fa8981d7b139a083d", "model": "OpAgent", "organization": "", "configuration": "OpAgent", "score": 0.716, "raw_score": 71.6, "raw_scale": 100, "source_record": "OpAgent|OpAgent", "source_date": "01/2026", "report_url": "https://github.com/codefuse-ai/Oagent" }, { "id": "7bd5b01b033f4f0124af", "model": "ColorBrowserAgent", "organization": "", "configuration": "ColorBrowserAgent; ite-specific knowledge adaptation & progressive summarization for long-horizon tasks.", "score": 0.7120000000000001, "raw_score": 71.2, "raw_scale": 100, "source_record": "ColorBrowserAgent|ColorBrowserAgent; ite-specific knowledge adaptation & progressive summarization for long-horizon tasks.", "source_date": "12/2025", "report_url": "https://github.com/MadeAgents/browser-agent" }, { "id": "857023c9b522185b1938", "model": "Claude Code + GBOX MCP", "organization": "Anthropic", "configuration": "GBOX AI", "score": 0.68, "raw_score": 68.0, "raw_scale": 100, "source_record": "Claude Code + GBOX MCP|GBOX AI", "source_date": "10/2025", "report_url": "https://gbox.ai/" }, { "id": "d206635bd44cf8811df7", "model": "DeepSky Agent", "organization": "", "configuration": "DeepSky Agent", "score": 0.669, "raw_score": 66.9, "raw_scale": 100, "source_record": "DeepSky Agent|DeepSky Agent", "source_date": "09/2025" }, { "id": "5a2e611f053ca7977f11", "model": "Narada AI", "organization": "", "configuration": "Narada AI", "score": 0.642, "raw_score": 64.2, "raw_scale": 100, "source_record": "Narada AI|Narada AI", "source_date": "10/2025" }, { "id": "085cd12098e055caf001", "model": "IBM CUGA", "organization": "", "configuration": "IBM CUGA", "score": 0.617, "raw_score": 61.7, "raw_scale": 100, "source_record": "IBM CUGA|IBM CUGA", "source_date": "02/2025", "report_url": "https://github.com/cuga-project/cuga-agent" }, { "id": "2c95658365d18bd4da65", "model": "OpenAI Operator", "organization": "", "configuration": "OpenAI CUA; System card", "score": 0.581, "raw_score": 58.1, "raw_scale": 100, "source_record": "OpenAI Operator|OpenAI CUA; System card", "source_date": "01/2025", "report_url": "https://openai.com/index/computer-using-agent/" }, { "id": "2f817e38ea8c4e4c85f6", "model": "Jace.AI", "organization": "", "configuration": "https://www.jace.ai/; Note from the developer of the work, see the comment of the cell", "score": 0.5710000000000001, "raw_score": 57.1, "raw_scale": 100, "source_record": "Jace.AI|https://www.jace.ai/; Note from the developer of the work, see the comment of the cell", "source_date": "08/2024" }, { "id": "9496324ab4fef577bdbd", "model": "GLM-4-32B + Qwen2.5-VL-7B", "organization": "Z AI", "configuration": "WebChallenger", "score": 0.563, "raw_score": 56.3, "raw_scale": 100, "source_record": "GLM-4-32B + Qwen2.5-VL-7B|WebChallenger", "source_date": "06/2026", "report_url": "https://arxiv.org/abs/2606.10423" }, { "id": "4052aa2463405695a22a", "model": "WebOperator + GPT-4o", "organization": "", "configuration": "WebOperator", "score": 0.546, "raw_score": 54.6, "raw_scale": 100, "source_record": "WebOperator + GPT-4o|WebOperator", "source_date": "12/2025", "report_url": "https://arxiv.org/abs/2512.12692" }, { "id": "9808f7df3efbc983025b", "model": "ScribeAgent + GPT-4o", "organization": "", "configuration": "ScribeAgent; ScribeAgent is finetuned with proprietary data", "score": 0.53, "raw_score": 53.0, "raw_scale": 100, "source_record": "ScribeAgent + GPT-4o|ScribeAgent; ScribeAgent is finetuned with proprietary data", "source_date": "12/2024", "report_url": "https://arxiv.org/abs/2411.15004" }, { "id": "962d6a8defa06d4c5b11", "model": "AgentSymbiotic", "organization": "", "configuration": "AgentSymbiotic; Code", "score": 0.521, "raw_score": 52.1, "raw_scale": 100, "source_record": "AgentSymbiotic|AgentSymbiotic; Code", "source_date": "01/2025", "report_url": "https://www.arxiv.org/abs/2502.07942" }, { "id": "b224041102f89f0cdda5", "model": "Learn-by-Interact", "organization": "", "configuration": "Learn-by-interact", "score": 0.48, "raw_score": 48.0, "raw_scale": 100, "source_record": "Learn-by-Interact|Learn-by-interact", "source_date": "01/2025", "report_url": "https://arxiv.org/abs/2501.10893" }, { "id": "4454346e6a379369e182", "model": "AgentOccam-Judge", "organization": "", "configuration": "AgentOccam-Judge", "score": 0.457, "raw_score": 45.7, "raw_scale": 100, "source_record": "AgentOccam-Judge|AgentOccam-Judge", "source_date": "10/2024", "report_url": "https://arxiv.org/abs/2410.13825" }, { "id": "a440de550eb3fd9ab78e", "model": "A3-Qwen3.5-9B", "organization": "", "configuration": "Agent-as-Annotators; Multimodal (AXTree + screenshots). SFT on 2,322 synthetic trajectories from Gemini 3 Pro. Evaluated using GenericAgent/BrowserGym.", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "A3-Qwen3.5-9B|Agent-as-Annotators; Multimodal (AXTree + screenshots). SFT on 2,322 synthetic trajectories from Gemini 3 Pro. Evaluated using GenericAgent/BrowserGym.", "source_date": "04/2026", "report_url": "https://arxiv.org/abs/2604.07776" }, { "id": "a4493a2e67a3f099e84d", "model": "WebPilot", "organization": "", "configuration": "WebPilot; No open source code or trajectory released from the work", "score": 0.37200000000000005, "raw_score": 37.2, "raw_scale": 100, "source_record": "WebPilot|WebPilot; No open source code or trajectory released from the work", "source_date": "08/2024", "report_url": "https://www.arxiv.org/pdf/2408.15978" }, { "id": "eb66ff93790fae1b7095", "model": "GUI-API Hybrid Agent", "organization": "", "configuration": "Beyond Browsing; Using both API and GUI", "score": 0.358, "raw_score": 35.8, "raw_scale": 100, "source_record": "GUI-API Hybrid Agent|Beyond Browsing; Using both API and GUI", "source_date": "10/2024", "report_url": "https://arxiv.org/pdf/2410.16464" }, { "id": "5afd14d33d0191a9e956", "model": "Agent Workflow Memory", "organization": "", "configuration": "AWM", "score": 0.355, "raw_score": 35.5, "raw_scale": 100, "source_record": "Agent Workflow Memory|AWM", "source_date": "09/2024", "report_url": "https://arxiv.org/pdf/2409.07429" }, { "id": "cdbc029190664cc4225b", "model": "SteP", "organization": "", "configuration": "SteP; High-level plans are derived by human", "score": 0.335, "raw_score": 33.5, "raw_scale": 100, "source_record": "SteP|SteP; High-level plans are derived by human", "source_date": "04/2024", "report_url": "https://arxiv.org/abs/2310.03720" }, { "id": "f1cfe34e7c22206bad33", "model": "TTI", "organization": "", "configuration": "TTI", "score": 0.261, "raw_score": 26.1, "raw_scale": 100, "source_record": "TTI|TTI", "source_date": "06/2025", "report_url": "https://huggingface.co/sjunhongs/tti_webarena/tree/main" }, { "id": "5f5c012b453232c8d4ff", "model": "BrowserGym + GPT-4", "organization": "", "configuration": "BrowserGym; different observation representation", "score": 0.235, "raw_score": 23.5, "raw_scale": 100, "source_record": "BrowserGym + GPT-4|BrowserGym; different observation representation", "source_date": "04/2024", "report_url": "https://arxiv.org/pdf/2403.07718.pdf" }, { "id": "28d149018cad3ef04c9d", "model": "AgentTrek-1.0-32B", "organization": "", "configuration": "AgentTrek", "score": 0.22399999999999998, "raw_score": 22.4, "raw_scale": 100, "source_record": "AgentTrek-1.0-32B|AgentTrek", "source_date": "01/2025", "report_url": "https://agenttrek.github.io/" }, { "id": "0cc451457df454c3deca", "model": "GPT-4 + Auto Eval", "organization": "OpenAI", "configuration": "Auto Eval & Refine", "score": 0.20199999999999999, "raw_score": 20.2, "raw_scale": 100, "source_record": "GPT-4 + Auto Eval|Auto Eval & Refine", "source_date": "04/2024", "report_url": "https://arxiv.org/abs/2404.06474" }, { "id": "f87d16b08eef0e166152", "model": "GPT-4o + Tree Search", "organization": "OpenAI", "configuration": "Tree Search for LM Agents", "score": 0.192, "raw_score": 19.2, "raw_scale": 100, "source_record": "GPT-4o + Tree Search|Tree Search for LM Agents", "source_date": "06/2024", "report_url": "https://jykoh.com/search-agents" }, { "id": "a1ea681367b673387337", "model": "AutoWebGLM", "organization": "", "configuration": "AutoWebGLM", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "AutoWebGLM|AutoWebGLM", "source_date": "04/2024", "report_url": "https://arxiv.org/abs/2404.03648" }, { "id": "1c20aa89bc04ebf40b7c", "model": "NNetNav", "organization": "", "configuration": "NNetscape; LLama 3.1-8B-instruct fine-tuned on NNetNav6k (a newer version of the dataset where the work keeps the best of 3 trajectories for each instruction, where we use a llama 3.1 70b as the reward model). The model is available here: https://huggingface.co/stanfordnlp/llama8b-nnetnav-wa", "score": 0.163, "raw_score": 16.3, "raw_scale": 100, "source_record": "NNetNav|NNetscape; LLama 3.1-8B-instruct fine-tuned on NNetNav6k (a newer version of the dataset where the work keeps the best of 3 trajectories for each instruction, where we use a llama 3.1 70b as the reward model). The model is available here: https://huggingface.co/stanfordnlp/llama8b-nnetnav-wa", "source_date": "01/2025", "report_url": "https://arxiv.org/abs/2410.02907" }, { "id": "620dab33c023cb96a2ab", "model": "gpt-4-0613", "organization": "OpenAI", "configuration": "GPT; when \"not achievable\" hint is not provided", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "gpt-4-0613|GPT; when \"not achievable\" hint is not provided", "source_date": "06/2023", "report_url": "https://arxiv.org/abs/2307.13854" }, { "id": "6708c3e37b23e3b809bf", "model": "gpt-4o-2024-05-13", "organization": "OpenAI", "configuration": "GPT; when \"not achievable\" hint is provided", "score": 0.131, "raw_score": 13.1, "raw_scale": 100, "source_record": "gpt-4o-2024-05-13|GPT; when \"not achievable\" hint is provided", "source_date": "05/2024" }, { "id": "f0d41ee02ea9d14a347a", "model": "gpt-4-0613", "organization": "OpenAI", "configuration": "GPT; when \"not achievable\" hint is provided", "score": 0.11699999999999999, "raw_score": 11.7, "raw_scale": 100, "source_record": "gpt-4-0613|GPT; when \"not achievable\" hint is provided", "source_date": "06/2023", "report_url": "https://arxiv.org/abs/2307.13854" }, { "id": "733216943cdcdf61aa8f", "model": "Qwen1.5-72B-Chat (Mixture B fine-tune)", "organization": "Alibaba", "configuration": "Patel et al + 2024", "score": 0.09359999999999999, "raw_score": 9.36, "raw_scale": 100, "source_record": "Patel et al + 2024|Patel et al + 2024", "source_date": "05/2024", "report_url": "https://arxiv.org/abs/2405.20309" }, { "id": "1f41e35aa4b87fb260fd", "model": "gpt-3.5-turbo-16k-0613", "organization": "OpenAI", "configuration": "GPT", "score": 0.08869999999999999, "raw_score": 8.87, "raw_scale": 100, "source_record": "gpt-3.5-turbo-16k-0613|GPT", "source_date": "03/2023", "report_url": "https://arxiv.org/abs/2307.13854" }, { "id": "0437fff81e6632b42de5", "model": "Qwen-1.5-chat-72b", "organization": "Alibaba", "configuration": "Qwen", "score": 0.07139999999999999, "raw_score": 7.14, "raw_scale": 100, "source_record": "Qwen-1.5-chat-72b|Qwen", "source_date": "09/2023", "report_url": "https://arxiv.org/abs/2405.20309" }, { "id": "8b9407abdae395810793", "model": "Gemini Pro", "organization": "Google", "configuration": "Gemini Pro", "score": 0.0712, "raw_score": 7.12, "raw_scale": 100, "source_record": "Gemini Pro|Gemini Pro", "source_date": "12/2023", "report_url": "https://arxiv.org/abs/2307.13854" }, { "id": "82e9556126a70a4ea6f5", "model": "Llama3-chat-70b", "organization": "Meta", "configuration": "Llama3", "score": 0.0702, "raw_score": 7.02, "raw_scale": 100, "source_record": "Llama3-chat-70b|Llama3", "source_date": "04/2024" }, { "id": "455fa37babd2f06b927d", "model": "Synatra-CodeLLama7b", "organization": "", "configuration": "Synatra", "score": 0.06280000000000001, "raw_score": 6.28, "raw_scale": 100, "source_record": "Synatra-CodeLLama7b|Synatra", "source_date": "10/2024", "report_url": "https://arxiv.org/pdf/2409.15637" }, { "id": "5fd387095699c782c49b", "model": "Lemur-chat-70b", "organization": "", "configuration": "Lemur", "score": 0.053, "raw_score": 5.3, "raw_scale": 100, "source_record": "Lemur-chat-70b|Lemur", "source_date": "10/2023", "report_url": "https://arxiv.org/abs/2310.06830" }, { "id": "31666795856996e7e598", "model": "Agent Flan", "organization": "", "configuration": "Agent Flan", "score": 0.046799999999999994, "raw_score": 4.68, "raw_scale": 100, "source_record": "Agent Flan|Agent Flan", "source_date": "03/2024", "report_url": "https://arxiv.org/abs/2403.12881" }, { "id": "29622eb06feb08a3bb5f", "model": "CodeLlama-instruct-34b", "organization": "", "configuration": "Llama2", "score": 0.0406, "raw_score": 4.06, "raw_scale": 100, "source_record": "CodeLlama-instruct-34b|Llama2", "source_date": "08/2023", "report_url": "https://arxiv.org/abs/2310.06830" }, { "id": "ff961910994c031e7efa", "model": "AgentLM-70b", "organization": "", "configuration": "Agent Tuning", "score": 0.0381, "raw_score": 3.81, "raw_scale": 100, "source_record": "AgentLM-70b|Agent Tuning", "source_date": "10/2023", "report_url": "https://arxiv.org/abs/2310.12823" }, { "id": "efb8d5d9bcd98534e3f8", "model": "Llama3-chat-8b", "organization": "Meta", "configuration": "Llama3", "score": 0.0332, "raw_score": 3.32, "raw_scale": 100, "source_record": "Llama3-chat-8b|Llama3", "source_date": "04/2024" }, { "id": "c920eba2426d376d9447", "model": "CodeAct Agent", "organization": "", "configuration": "CodeAct", "score": 0.023, "raw_score": 2.3, "raw_scale": 100, "source_record": "CodeAct Agent|CodeAct", "source_date": "02/2024" }, { "id": "38ff832e755b970b76c3", "model": "AgentLM-13b", "organization": "", "configuration": "Agent Tuning", "score": 0.016, "raw_score": 1.6, "raw_scale": 100, "source_record": "AgentLM-13b|Agent Tuning", "source_date": "10/2023", "report_url": "https://arxiv.org/abs/2310.12823" }, { "id": "e480da6f29996873ea61", "model": "Mixtral", "organization": "Mistral AI", "configuration": "Mixtral", "score": 0.0139, "raw_score": 1.39, "raw_scale": 100, "source_record": "Mixtral|Mixtral", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2307.13854" }, { "id": "2e21c8206dbc0a5946b5", "model": "AgentLM-7b", "organization": "", "configuration": "Agent Tuning", "score": 0.0074, "raw_score": 0.74, "raw_scale": 100, "source_record": "AgentLM-7b|Agent Tuning", "source_date": "10/2023", "report_url": "https://arxiv.org/abs/2310.12823" }, { "id": "cb04570b01c68f89ab35", "model": "FireAct", "organization": "", "configuration": "FireAct", "score": 0.0025, "raw_score": 0.25, "raw_scale": 100, "source_record": "FireAct|FireAct", "source_date": "10/2023", "report_url": "https://arxiv.org/abs/2403.12881" }, { "id": "f129aacdab4e5ff09d03", "model": "CodeLlama-instruct-7b", "organization": "", "configuration": "CodeLLama", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "CodeLlama-instruct-7b|CodeLLama", "source_date": "08/2023" } ] }, { "id": "visualwebarena", "name": "VisualWebArena", "group": "Agent 与工具", "unit": "%", "metric": "Task success", "protocol": { "en": "Official submission leaderboard; agent scaffolds and input modes vary and are retained. Scores are reported by submitting teams.", "zh": "官方收录的提交榜单；Agent 框架与输入模式可能不同，随结果保留。成绩由提交团队报告。" }, "source": { "data_url": "https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ/export?format=csv&gid=2044883967", "sha256": "be851ca44c7141bf0f4acfb6e1844955f4d82c331df3b603f3531e3f0c7a74ee", "retrieved_at": "2026-11-02T05:57:39.137257+00:00", "name": "VisualWebArena authors", "url": "https://jykoh.com/vwa/", "selector": "Official linked submission sheet; Success Rate(%); exclude human/random baselines", "status": "current_snapshot", "kind": "official_submissions", "artifacts": [ { "data_url": "https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ/export?format=xlsx", "sha256": "0aceb7e0a4b90bb8f2b9076c436585c2cd8bd900642765c407d1bbf55296517d", "retrieved_at": "2026-11-02T05:59:32.071345+00:00" } ] }, "results": [ { "id": "0fe9cd2b1a4b6024aea7", "model": "Gemini 2.5 Flash", "organization": "Google", "configuration": "Multimodal (SoM); SoM; SGV", "score": 0.54, "raw_score": 54.0, "raw_scale": 100, "source_record": "Gemini 2.5 Flash|Multimodal (SoM); SoM; SGV", "source_date": "07/2025", "report_url": "https://arxiv.org/abs/2507.11662" }, { "id": "f592b376fb5a29217218", "model": "GPT-5", "organization": "OpenAI", "configuration": "Multimodal (SoM); SoM; WALT", "score": 0.529, "raw_score": 52.9, "raw_scale": 100, "source_record": "GPT-5|Multimodal (SoM); SoM; WALT", "source_date": "10/2025", "report_url": "https://arxiv.org/abs/2510.01524" }, { "id": "c9824361c9f2cb469c92", "model": "GLM-4-32B", "organization": "Z AI", "configuration": "Caption-augmented; Structured DOM Representation; WebChallenger", "score": 0.48700000000000004, "raw_score": 48.7, "raw_scale": 100, "source_record": "GLM-4-32B|Caption-augmented; Structured DOM Representation; WebChallenger", "source_date": "06/2026", "report_url": "https://arxiv.org/abs/2606.10423" }, { "id": "351983155df52ce9d83f", "model": "GPT-5", "organization": "OpenAI", "configuration": "Multimodal (SoM); SoM + Caption + Image; AWorld", "score": 0.365, "raw_score": 36.5, "raw_scale": 100, "source_record": "GPT-5|Multimodal (SoM); SoM + Caption + Image; AWorld", "source_date": "09/2025" }, { "id": "55aa0821fd8a954e3df1", "model": "A3-Qwen3.5-9B", "organization": "", "configuration": "Multimodal; AXTree + Image; Agent-as-Annotators", "score": 0.337, "raw_score": 33.7, "raw_scale": 100, "source_record": "A3-Qwen3.5-9B|Multimodal; AXTree + Image; Agent-as-Annotators", "source_date": "04/2026", "report_url": "https://arxiv.org/abs/2604.07776" }, { "id": "62624ae502d9a1b059ce", "model": "GPT-4o + R-MCTS", "organization": "OpenAI", "configuration": "Multimodal (SoM); SoM + Caption + Image; ExACT", "score": 0.337, "raw_score": 33.7, "raw_scale": 100, "source_record": "GPT-4o + R-MCTS|Multimodal (SoM); SoM + Caption + Image; ExACT", "source_date": "10/2024", "report_url": "https://agent-e3.github.io/ExACT/" }, { "id": "8aa4c3409a8384fc63f4", "model": "GPT-4o + Search", "organization": "OpenAI", "configuration": "Multimodal (SoM); SoM + Caption + Image; Tree Search for LM Agents", "score": 0.264, "raw_score": 26.4, "raw_scale": 100, "source_record": "GPT-4o + Search|Multimodal (SoM); SoM + Caption + Image; Tree Search for LM Agents", "source_date": "06/2024", "report_url": "https://jykoh.com/search-agents" }, { "id": "cd002f8425fae3c6cc4e", "model": "GPT-4o + WebDreamer", "organization": "OpenAI", "configuration": "Multimodal (SoM); SoM + Caption + Image; WebDreamer", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "GPT-4o + WebDreamer|Multimodal (SoM); SoM + Caption + Image; WebDreamer", "source_date": "11/2024", "report_url": "https://arxiv.org/abs/2411.06559" }, { "id": "a1530de82b74161671a8", "model": "GPT-4o + ICAL", "organization": "OpenAI", "configuration": "Multimodal (SoM); SoM + Caption + Image; ICAL", "score": 0.23399999999999999, "raw_score": 23.4, "raw_scale": 100, "source_record": "GPT-4o + ICAL|Multimodal (SoM); SoM + Caption + Image; ICAL", "source_date": "06/2024", "report_url": "https://arxiv.org/abs/2406.14596" }, { "id": "beb9e305d949920db7ed", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "score": 0.1978, "raw_score": 19.78, "raw_scale": 100, "source_record": "GPT-4o|Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "source_date": "06/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "9c8b512ebf43d2def0be", "model": "Llama-3-70B + Search", "organization": "Meta", "configuration": "Caption-augmented; AxTree + Caption; Tree Search for LM Agents", "score": 0.16699999999999998, "raw_score": 16.7, "raw_scale": 100, "source_record": "Llama-3-70B + Search|Caption-augmented; AxTree + Caption; Tree Search for LM Agents", "source_date": "06/2024", "report_url": "https://jykoh.com/search-agents" }, { "id": "e4fbca4781ecc83d47ad", "model": "GPT-4V", "organization": "OpenAI", "configuration": "Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "score": 0.1637, "raw_score": 16.37, "raw_scale": 100, "source_record": "GPT-4V|Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "80e3a628df6452ca6afd", "model": "GPT-4V", "organization": "OpenAI", "configuration": "Multimodal; AXTree + Caption + Image; VisualWebArena", "score": 0.1505, "raw_score": 15.05, "raw_scale": 100, "source_record": "GPT-4V|Multimodal; AXTree + Caption + Image; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "88a92459d03693689a8e", "model": "GPT-4 + BLIP-2-T5XL", "organization": "OpenAI", "configuration": "Caption-augmented; AxTree + Caption; VisualWebArena", "score": 0.1275, "raw_score": 12.75, "raw_scale": 100, "source_record": "GPT-4 + BLIP-2-T5XL|Caption-augmented; AxTree + Caption; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "1a265520d377230822e3", "model": "Gemini-Pro-1.5", "organization": "Google", "configuration": "Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "score": 0.1198, "raw_score": 11.98, "raw_scale": 100, "source_record": "Gemini-Pro-1.5|Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "source_date": "06/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "23d0d1a8b8429a07cefb", "model": "ViGoRL-7B", "organization": "", "configuration": "Image-only; Image; ViGoRL", "score": 0.11199999999999999, "raw_score": 11.2, "raw_scale": 100, "source_record": "ViGoRL-7B|Image-only; Image; ViGoRL", "source_date": "05/2025", "report_url": "https://arxiv.org/abs/2505.23678" }, { "id": "cd550b99746b04709338", "model": "Llama-3-70B-Instruct + BLIP-2-T5XL", "organization": "Meta", "configuration": "Caption-augmented; AxTree + Caption; VisualWebArena", "score": 0.0978, "raw_score": 9.78, "raw_scale": 100, "source_record": "Llama-3-70B-Instruct + BLIP-2-T5XL|Caption-augmented; AxTree + Caption; VisualWebArena", "source_date": "06/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "bd990c7e7617f4037fc7", "model": "Qwen2-VL + ICAL", "organization": "Alibaba", "configuration": "Multimodal (SoM); SoM + Caption + Image; ICAL", "score": 0.08199999999999999, "raw_score": 8.2, "raw_scale": 100, "source_record": "Qwen2-VL + ICAL|Multimodal (SoM); SoM + Caption + Image; ICAL", "source_date": "01/2025", "report_url": "https://arxiv.org/abs/2406.14596" }, { "id": "29ee209e3c7db559c513", "model": "GPT-4", "organization": "OpenAI", "configuration": "Text-only; AXTree; VisualWebArena", "score": 0.0725, "raw_score": 7.25, "raw_scale": 100, "source_record": "GPT-4|Text-only; AXTree; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "3d95e89b468778c9001b", "model": "Gemini-Flash-1.5", "organization": "Google", "configuration": "Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "score": 0.0659, "raw_score": 6.59, "raw_scale": 100, "source_record": "Gemini-Flash-1.5|Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "source_date": "06/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "920ce431e8c02674503b", "model": "ViGoRL-3B", "organization": "", "configuration": "Image-only; Image; ViGoRL", "score": 0.064, "raw_score": 6.4, "raw_scale": 100, "source_record": "ViGoRL-3B|Image-only; Image; ViGoRL", "source_date": "05/2025", "report_url": "https://arxiv.org/abs/2505.23678" }, { "id": "d5c8dca2193d3b70ec3e", "model": "Gemini-Pro", "organization": "Google", "configuration": "Multimodal; AXTree + Caption + Image; VisualWebArena", "score": 0.0604, "raw_score": 6.04, "raw_scale": 100, "source_record": "Gemini-Pro|Multimodal; AXTree + Caption + Image; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "bfee6569f1623faacf46", "model": "Gemini-Pro", "organization": "Google", "configuration": "Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "score": 0.0571, "raw_score": 5.71, "raw_scale": 100, "source_record": "Gemini-Pro|Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "bb03e7de92ec29fb9bab", "model": "Qwen-VL-7B", "organization": "Alibaba", "configuration": "Image-only; Image; ViGoRL", "score": 0.055, "raw_score": 5.5, "raw_scale": 100, "source_record": "Qwen-VL-7B|Image-only; Image; ViGoRL", "source_date": "05/2025", "report_url": "https://arxiv.org/abs/2505.23678" }, { "id": "abac5f9dfefdd2ba1725", "model": "Gemini-Pro + BLIP-2-T5XL", "organization": "Google", "configuration": "Caption-augmented; AxTree + Caption; VisualWebArena", "score": 0.0385, "raw_score": 3.85, "raw_scale": 100, "source_record": "Gemini-Pro + BLIP-2-T5XL|Caption-augmented; AxTree + Caption; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "b8cd49ce1b33f1ea3d0b", "model": "GPT-3.5 + BLIP-2-T5XL", "organization": "OpenAI", "configuration": "Caption-augmented; AxTree + Caption; VisualWebArena", "score": 0.0297, "raw_score": 2.97, "raw_scale": 100, "source_record": "GPT-3.5 + BLIP-2-T5XL|Caption-augmented; AxTree + Caption; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "888b42157f943a6779c1", "model": "Qwen2-VL", "organization": "Alibaba", "configuration": "Multimodal (SoM); SoM + Caption + Image; ICAL", "score": 0.028999999999999998, "raw_score": 2.9, "raw_scale": 100, "source_record": "Qwen2-VL|Multimodal (SoM); SoM + Caption + Image; ICAL", "source_date": "01/2025", "report_url": "https://arxiv.org/abs/2406.14596" }, { "id": "6d02fc251f87a900901a", "model": "GPT-3.5 + LLaVa-7B", "organization": "OpenAI", "configuration": "Caption-augmented; AxTree + Caption; VisualWebArena", "score": 0.0275, "raw_score": 2.75, "raw_scale": 100, "source_record": "GPT-3.5 + LLaVa-7B|Caption-augmented; AxTree + Caption; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "171aaafb1db4ed9c6759", "model": "GPT-3.5", "organization": "OpenAI", "configuration": "Text-only; AXTree; VisualWebArena", "score": 0.022000000000000002, "raw_score": 2.2, "raw_scale": 100, "source_record": "GPT-3.5|Text-only; AXTree; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "6446e4b442f77608d71e", "model": "Gemini-Pro", "organization": "Google", "configuration": "Text-only; AXTree; VisualWebArena", "score": 0.022000000000000002, "raw_score": 2.2, "raw_scale": 100, "source_record": "Gemini-Pro|Text-only; AXTree; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "60ad7f797bf3831f4eaa", "model": "Mixtral-8x7b + BLIP-2-T5XL", "organization": "Mistral AI", "configuration": "Caption-augmented; AxTree + Caption; VisualWebArena", "score": 0.0187, "raw_score": 1.87, "raw_scale": 100, "source_record": "Mixtral-8x7b + BLIP-2-T5XL|Caption-augmented; AxTree + Caption; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "6a0262a60cb7f84b6b9f", "model": "Mixtral-8x7B", "organization": "Mistral AI", "configuration": "Text-only; AXTree; VisualWebArena", "score": 0.0176, "raw_score": 1.76, "raw_scale": 100, "source_record": "Mixtral-8x7B|Text-only; AXTree; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "b3a1e4ca6eb861f27cf8", "model": "Llama-2-70B", "organization": "Meta", "configuration": "Text-only; AXTree; VisualWebArena", "score": 0.011000000000000001, "raw_score": 1.1, "raw_scale": 100, "source_record": "Llama-2-70B|Text-only; AXTree; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "a578659825135f935bac", "model": "IDEFICS-80B-Instruct", "organization": "", "configuration": "Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "score": 0.009899999999999999, "raw_score": 0.99, "raw_scale": 100, "source_record": "IDEFICS-80B-Instruct|Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "45d60b108157728a9e9e", "model": "IDEFICS-80B-Instruct", "organization": "", "configuration": "Multimodal; AXTree + Caption + Image; VisualWebArena", "score": 0.0077, "raw_score": 0.77, "raw_scale": 100, "source_record": "IDEFICS-80B-Instruct|Multimodal; AXTree + Caption + Image; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "f0ac62f42d504676cf1d", "model": "Llama-2-70B + BLIP-2-T5XL", "organization": "Meta", "configuration": "Caption-augmented; AxTree + Caption; VisualWebArena", "score": 0.0066, "raw_score": 0.66, "raw_scale": 100, "source_record": "Llama-2-70B + BLIP-2-T5XL|Caption-augmented; AxTree + Caption; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "9e29e0283fe0f29afcd5", "model": "CogVLM", "organization": "", "configuration": "Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "score": 0.0033, "raw_score": 0.33, "raw_scale": 100, "source_record": "CogVLM|Multimodal (SoM); SoM + Caption + Image; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" }, { "id": "faccd7989a6346f43090", "model": "CogVLM", "organization": "", "configuration": "Multimodal; AXTree + Caption + Image; VisualWebArena", "score": 0.0033, "raw_score": 0.33, "raw_scale": 100, "source_record": "CogVLM|Multimodal; AXTree + Caption + Image; VisualWebArena", "source_date": "01/2024", "report_url": "https://arxiv.org/abs/2401.13649" } ] }, { "id": "androidworld", "name": "AndroidWorld", "group": "Agent 与工具", "unit": "%", "metric": "Task success (pass@1)", "protocol": { "en": "Official community-submission board, not independently verified. Uses pass@1 only; trial counts, agents and screen representations are retained.", "zh": "官方社区提交榜单，发布方未独立验证。仅使用 pass@1；保留试验次数、Agent 和屏幕表示。" }, "source": { "data_url": "https://docs.google.com/spreadsheets/d/1cchzP9dlTZ3WXQTfYNhh3avxoLipqHN75v1Tb86uhHo/export?format=csv&gid=0", "sha256": "fa55b5154926e6027efa02dda6a07d15e008b3e314adbc20d0980ce537aca8f0", "retrieved_at": "2026-11-02T05:57:39.702107+00:00", "name": "Google Research / AndroidWorld", "url": "https://google-research.github.io/android_world/", "selector": "Named model; Success Rate(pass@1); exclude Human, missing models and pass@k-only submissions; preserve trial count", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "b64ae0f7baa03b761584", "model": "AGI-AW", "organization": "", "configuration": "AGI-0; Screenshot; [09/2026]: 97.4 --> 100. Trained open source models using our novel training technique on AndroidWorld tasks. Code and model will be open-sourced upon publication.; trials: 1", "score": 1.0, "raw_score": 100.0, "raw_scale": 100, "source_record": "AGI-AW|AGI-0; Screenshot; [09/2026]: 97.4 --> 100. Trained open source models using our novel training technique on AndroidWorld tasks. Code and model will be open-sourced upon publication.; trials: 1", "source_date": "10/2025" }, { "id": "d34c904a8979d7e66572", "model": "GPT6 Astra", "organization": "OpenAI", "configuration": "MobileRun; Screenshot + A11y tree; [09/30/2026] 91.4 --> Updated harness and using GPT-6 Astra [10/6/2025]: 78.4 --> 91.4; details here [09/19/2025]: Updated score from 63.0 --> 78.4. For this run, we used GPT-5 for reasoning in combination with Gemini-2.5-Pro for acting, while continuing to rely on a hybrid method of the Accessibility (A11y) tree plus corresponding screenshots to provide both structural and visual context to the agent.; trials: 1", "score": 1.0, "raw_score": 100.0, "raw_scale": 100, "source_record": "GPT6 Astra|MobileRun; Screenshot + A11y tree; [09/30/2026] 91.4 --> Updated harness and using GPT-6 Astra [10/6/2025]: 78.4 --> 91.4; details here [09/19/2025]: Updated score from 63.0 --> 78.4. For this run, we used GPT-5 for reasoning in combination with Gemini-2.5-Pro for acting, while continuing to rely on a hybrid method of the Accessibility (A11y) tree plus corresponding screenshots to provide both structural and visual context to the agent.; trials: 1", "source_date": "09/2026" }, { "id": "a0fb67eb66b94acb8bea", "model": "gpt-4o / gpt-5.4", "organization": "OpenAI", "configuration": "Kirk Engine; Screenshot + A11y tree; Code is not open-sourced yet.; trials: 1", "score": 1.0, "raw_score": 100.0, "raw_scale": 100, "source_record": "gpt-4o / gpt-5.4|Kirk Engine; Screenshot + A11y tree; Code is not open-sourced yet.; trials: 1", "source_date": "08/2026", "report_url": "https://abhishekdh.github.io/kirk-engine/" }, { "id": "2aebd4767281e4ea03bd", "model": "gpt-4o / gpt-5.6-sol", "organization": "OpenAI", "configuration": "FluizAI; Screenshot + A11y tree; Code is not open-sourced yet.; trials: 1", "score": 1.0, "raw_score": 100.0, "raw_scale": 100, "source_record": "gpt-4o / gpt-5.6-sol|FluizAI; Screenshot + A11y tree; Code is not open-sourced yet.; trials: 1", "source_date": "08/2026", "report_url": "https://android-world.fluiz.ai/" }, { "id": "d1f8fe3de4a455bf52f9", "model": "gpt-6-sol", "organization": "OpenAI", "configuration": "agent-qa by Vostride AI; Screenshot + A11y tree; Note: Open-source self-improving QA agent with memory. Code is available at https://github.com/vostride/agent-qa. agent-qa's memory and self-improvement loop were switched off during the benchmark run.; trials: 1", "score": 1.0, "raw_score": 100.0, "raw_scale": 100, "source_record": "gpt-6-sol|agent-qa by Vostride AI; Screenshot + A11y tree; Note: Open-source self-improving QA agent with memory. Code is available at https://github.com/vostride/agent-qa. agent-qa's memory and self-improvement loop were switched off during the benchmark run.; trials: 1", "source_date": "09/2026", "report_url": "https://vostride.com/benchmark/android-world" }, { "id": "b0016166868502009a9f", "model": "Gemini 3.7 Flash, Gemini Robotics ER 2", "organization": "Google", "configuration": "Artemis; Screenshot + A11y tree; Open-source project available at https://github.com/google/artemis; trials: 1", "score": 0.991, "raw_score": 99.1, "raw_scale": 100, "source_record": "Gemini 3.7 Flash, Gemini Robotics ER 2|Artemis; Screenshot + A11y tree; Open-source project available at https://github.com/google/artemis; trials: 1", "source_date": "08/2026" }, { "id": "29bee46f85e4cd85e8d8", "model": "Gemini 3 flash, Gemini 3 flash lite", "organization": "Google", "configuration": "Finalrun; Screenshot + A11y tree; 04/2026: Project is open-source at https://github.com/final-run/finalrun-agent 03/2026: Updated results: https://blogs.finalrun.app/finalrun-achieves-the-new-1-sota-mobile-agent-eclipsing-benchmarks-at-97-4 08/2025: The technical doc/approach can be found here: https://github.com/final-run/finalrun-android-world-benchmark; trials: 1", "score": 0.9740000000000001, "raw_score": 97.4, "raw_scale": 100, "source_record": "Gemini 3 flash, Gemini 3 flash lite|Finalrun; Screenshot + A11y tree; 04/2026: Project is open-source at https://github.com/final-run/finalrun-agent 03/2026: Updated results: https://blogs.finalrun.app/finalrun-achieves-the-new-1-sota-mobile-agent-eclipsing-benchmarks-at-97-4 08/2025: The technical doc/approach can be found here: https://github.com/final-run/finalrun-android-world-benchmark; trials: 1", "source_date": "3/2026", "report_url": "https://drive.google.com/file/d/1NITEkiYb3itSq46I8BYsmzefEddzYbqm/view?usp=sharing" }, { "id": "b42577300183b27aa42c", "model": "Seed1.8-GUI", "organization": "", "configuration": "MobileUseAgent; Screenshot; trials: 1", "score": 0.9740000000000001, "raw_score": 97.4, "raw_scale": 100, "source_record": "Seed1.8-GUI|MobileUseAgent; Screenshot; trials: 1", "source_date": "02/2026" }, { "id": "75297a1e84cb8d854ff2", "model": "askui AndroidVisionAgent, Claude 4.5 Sonnet + Claude 4.0 Sonnet", "organization": "", "configuration": "askui AndroidVisionAgent; Screenshot; trials: 1", "score": 0.948, "raw_score": 94.8, "raw_scale": 100, "source_record": "askui AndroidVisionAgent, Claude 4.5 Sonnet + Claude 4.0 Sonnet|askui AndroidVisionAgent; Screenshot; trials: 1", "source_date": "10/2025" }, { "id": "6926fbd6442af8044c93", "model": "gemini 3 pro + sonnet 4.5", "organization": "Google", "configuration": "AutoDevice; Screenshot; Agent code is available on https://github.com/autodevice/android_world; trials: 1", "score": 0.948, "raw_score": 94.8, "raw_scale": 100, "source_record": "gemini 3 pro + sonnet 4.5|AutoDevice; Screenshot; Agent code is available on https://github.com/autodevice/android_world; trials: 1", "source_date": "01/2026", "report_url": "https://autodevice.io/benchmark/" }, { "id": "47a3ebd9edf54cabc843", "model": "Gemini-3.5-Flash", "organization": "Google", "configuration": "Midscene.js; Screenshot; trials: 1", "score": 0.9309999999999999, "raw_score": 93.1, "raw_scale": 100, "source_record": "Gemini-3.5-Flash|Midscene.js; Screenshot; trials: 1", "source_date": "06/2026" }, { "id": "82fbebd0bcb59aaf90f4", "model": "Llama 4-scout, Gemini 2.5 pro, GPT-5 nano", "organization": "Meta", "configuration": "mobile-use; Screenshot + A11y tree; [12/16/2025] 84.5% -> 91.4% [10/1/2025]: 77.6% -> 84.5% [09/12/2025]: Updated score from 74.1% --> 77.6% [08/19/2025]: Initial trajectory labeling issue (MarkorCreateNoteFromClipboard) has been corrected by the authors. They provide Discord support for setup issues. [08/18/2025]: Repo has been reported broken by some users (not independently verified). Some trajectories (e.g., MarkorCreateNoteFromClipboard, BrowserMaze) are labeled as successful by the authors but contain incorrect actions; trials: 1", "score": 0.914, "raw_score": 91.4, "raw_scale": 100, "source_record": "Llama 4-scout, Gemini 2.5 pro, GPT-5 nano|mobile-use; Screenshot + A11y tree; [12/16/2025] 84.5% -> 91.4% [10/1/2025]: 77.6% -> 84.5% [09/12/2025]: Updated score from 74.1% --> 77.6% [08/19/2025]: Initial trajectory labeling issue (MarkorCreateNoteFromClipboard) has been corrected by the authors. They provide Discord support for setup issues. [08/18/2025]: Repo has been reported broken by some users (not independently verified). Some trajectories (e.g., MarkorCreateNoteFromClipboard, BrowserMaze) are labeled as successful by the authors but contain incorrect actions; trials: 1", "source_date": "9/2025", "report_url": "https://minitap.ai/benchmark" }, { "id": "2b371f4bd52c568f4dfb", "model": "o3 + holo1.5-72b", "organization": "", "configuration": "Surfer 2; Screenshot; Agent is based on https://github.com/hcompai/surfer-h-cli - same architecture, but with modifications to prompt and action space; trials: 1", "score": 0.871, "raw_score": 87.1, "raw_scale": 100, "source_record": "o3 + holo1.5-72b|Surfer 2; Screenshot; Agent is based on https://github.com/hcompai/surfer-h-cli - same architecture, but with modifications to prompt and action space; trials: 1", "source_date": "10/2025", "report_url": "https://hcompai.github.io/android-world-traces/" }, { "id": "5b081050a438f18f41e4", "model": "Sonnet 4.5 + Sonnet 4", "organization": "", "configuration": "gbox.ai; Screenshot; GBOX uses Claude code as the agent with GBOX mcp. Link to report -> https://github.com/babelcloud/android_world_benchmark/blob/main/GBOX/report.md; trials: 1", "score": 0.862, "raw_score": 86.2, "raw_scale": 100, "source_record": "Sonnet 4.5 + Sonnet 4|gbox.ai; Screenshot; GBOX uses Claude code as the agent with GBOX mcp. Link to report -> https://github.com/babelcloud/android_world_benchmark/blob/main/GBOX/report.md; trials: 1", "source_date": "10/2025", "report_url": "https://github.com/babelcloud/android_world_benchmark/tree/main/GBOX/trajectory" }, { "id": "abcdb8630151dba35376", "model": "AutoGLM-Mobile", "organization": "", "configuration": "AutoGLM-Mobile; Screenshot + A11y tree; [9/26/2025]: Updated score from 75.8 --> 80.2 This is an updated version of AutoGLM-Mobile. In this version, we added more pretraining data and trained the VLM at full image resolution.; trials: 1", "score": 0.802, "raw_score": 80.2, "raw_scale": 100, "source_record": "AutoGLM-Mobile|AutoGLM-Mobile; Screenshot + A11y tree; [9/26/2025]: Updated score from 75.8 --> 80.2 This is an updated version of AutoGLM-Mobile. In this version, we added more pretraining data and trained the VLM at full image resolution.; trials: 1", "source_date": "8/2025" }, { "id": "9f6723adfc833e8d7df6", "model": "LX-GUIAgent", "organization": "", "configuration": "LX-GUIAgent; Screenshot + A11y tree; [9/23/2025]: Updated score from 75.0 --> 79.3. Added trajectories.; trials: 1", "score": 0.7929999999999999, "raw_score": 79.3, "raw_scale": 100, "source_record": "LX-GUIAgent|LX-GUIAgent; Screenshot + A11y tree; [9/23/2025]: Updated score from 75.0 --> 79.3. Added trajectories.; trials: 1", "source_date": "9/2025" }, { "id": "9a85bbef8f25702fcd59", "model": "Gemini-2.5-Pro+UI-TARS-1.5", "organization": "Google", "configuration": "AgentProg; Screenshot; Paper: https://arxiv.org/pdf/2512.10371; Code: https://github.com/MobileLLM/AgentProg; trials: 1", "score": 0.78, "raw_score": 78.0, "raw_scale": 100, "source_record": "Gemini-2.5-Pro+UI-TARS-1.5|AgentProg; Screenshot; Paper: https://arxiv.org/pdf/2512.10371; Code: https://github.com/MobileLLM/AgentProg; trials: 1", "source_date": "12/2025" }, { "id": "ab0a8a029af8e97e6524", "model": "MAI-UI-235B-A22B", "organization": "", "configuration": "MAI-UI; Screenshot; trials: ", "score": 0.767, "raw_score": 76.7, "raw_scale": 100, "source_record": "MAI-UI-235B-A22B|MAI-UI; Screenshot; trials: ", "source_date": "1/2026" }, { "id": "392766459e27ea018fdd", "model": "Qwen2.5-VL-72B + Qwen2.5-VL-7B", "organization": "Alibaba", "configuration": "K²-Agent; Screenshot; K²-Agent, a hierarchical approach that self-evolves a Qwen2.5-VL-72B for high-level planning and post-trains a Qwen2.5-VL-7B for low-level execution. Our GitHub repository, which contains a detailed description of our approach, can be found here: https://github.com/k2-agent/k2-agent.; trials: 1", "score": 0.767, "raw_score": 76.7, "raw_scale": 100, "source_record": "Qwen2.5-VL-72B + Qwen2.5-VL-7B|K²-Agent; Screenshot; K²-Agent, a hierarchical approach that self-evolves a Qwen2.5-VL-72B for high-level planning and post-trains a Qwen2.5-VL-7B for low-level execution. Our GitHub repository, which contains a detailed description of our approach, can be found here: https://github.com/k2-agent/k2-agent.; trials: 1", "source_date": "9/2025", "report_url": "https://github.com/k2-agent/k2-agent/blob/main/androidworld-trajectories.zip" }, { "id": "ff2fa628ae3f0a328861", "model": "Hammer-UI-32B", "organization": "", "configuration": "MobileUse-v2; Screenshot; Make further post-training based on the GUI-Owl-32b model. Optimize the memory and knowledge module of the MobileUse framework.; trials: 1", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "Hammer-UI-32B|MobileUse-v2; Screenshot; Make further post-training based on the GUI-Owl-32b model. Optimize the memory and knowledge module of the MobileUse framework.; trials: 1", "source_date": "9/2025" }, { "id": "b408f746792820661cae", "model": "GUI-Owl-32B", "organization": "", "configuration": "Mobile-Agent-v3; Screenshot; https://github.com/X-PLUG/MobileAgent/tree/main; trials: 1", "score": 0.733, "raw_score": 73.3, "raw_scale": 100, "source_record": "GUI-Owl-32B|Mobile-Agent-v3; Screenshot; https://github.com/X-PLUG/MobileAgent/tree/main; trials: 1", "source_date": "8/2025" }, { "id": "348720530bd518158826", "model": "MAI-UI-32B", "organization": "", "configuration": "MAI-UI; Screenshot; trials: ", "score": 0.733, "raw_score": 73.3, "raw_scale": 100, "source_record": "MAI-UI-32B|MAI-UI; Screenshot; trials: ", "source_date": "1/2026" }, { "id": "eb35d1954d823072a6fd", "model": "MAI-UI-8B", "organization": "", "configuration": "MAI-UI; Screenshot; trials: ", "score": 0.7070000000000001, "raw_score": 70.7, "raw_scale": 100, "source_record": "MAI-UI-8B|MAI-UI; Screenshot; trials: ", "source_date": "1/2026" }, { "id": "c694aafefbd75565aa13", "model": "Gemini 2.5 Computer Use", "organization": "Google", "configuration": "Gemini 2.5 Computer Use; Screenshot; trials: 1", "score": 0.6970000000000001, "raw_score": 69.7, "raw_scale": 100, "source_record": "Gemini 2.5 Computer Use|Gemini 2.5 Computer Use; Screenshot; trials: 1", "source_date": "10/2025" }, { "id": "9636b8e66037a152b4b8", "model": "JT-GUIAgent-V2", "organization": "", "configuration": "JT-GUIAgent-V2; Screenshot; trials: 1", "score": 0.672, "raw_score": 67.2, "raw_scale": 100, "source_record": "JT-GUIAgent-V2|JT-GUIAgent-V2; Screenshot; trials: 1", "source_date": "6/2025" }, { "id": "178cf0e050e52c0f7c6e", "model": "GUI-Owl-7B", "organization": "", "configuration": "GUI-Owl-7B; Screenshot; https://github.com/X-PLUG/MobileAgent/tree/main; trials: 1", "score": 0.664, "raw_score": 66.4, "raw_scale": 100, "source_record": "GUI-Owl-7B|GUI-Owl-7B; Screenshot; https://github.com/X-PLUG/MobileAgent/tree/main; trials: 1", "source_date": "8/2025" }, { "id": "6fd0e6ff29827010ff29", "model": "UI-Venus-Navi-72B", "organization": "", "configuration": "UI-Venus; Screenshot; https://huggingface.co/inclusionAI/UI-Venus-Navi-72B; trials: 1", "score": 0.659, "raw_score": 65.9, "raw_scale": 100, "source_record": "UI-Venus-Navi-72B|UI-Venus; Screenshot; https://huggingface.co/inclusionAI/UI-Venus-Navi-72B; trials: 1", "source_date": "8/2025", "report_url": "https://github.com/inclusionAI/UI-Venus/blob/main/vis_androidworld/UI-Venus-androidworld.zip" }, { "id": "e9e652cce22401c69c2a", "model": "Qwen2.5-VL-72B", "organization": "Alibaba", "configuration": "MobileUse; Screenshot; trials: 1", "score": 0.629, "raw_score": 62.9, "raw_scale": 100, "source_record": "Qwen2.5-VL-72B|MobileUse; Screenshot; trials: 1", "source_date": "07/2025" }, { "id": "1a48eeaf29eb0b9bdb26", "model": "Seed1.5-VL", "organization": "", "configuration": "Seed1.5-VL; Screenshot + A11y tree; trials: 1", "score": 0.621, "raw_score": 62.1, "raw_scale": 100, "source_record": "Seed1.5-VL|Seed1.5-VL; Screenshot + A11y tree; trials: 1", "source_date": "05/2025" }, { "id": "a84cb2ea506a8a347123", "model": "JT-GUIAgent-V1", "organization": "", "configuration": "JT-GUIAgent-V1; Screenshot; trials: 1", "score": 0.6, "raw_score": 60.0, "raw_scale": 100, "source_record": "JT-GUIAgent-V1|JT-GUIAgent-V1; Screenshot; trials: 1", "source_date": "6/2025" }, { "id": "9edb7aabf18103208ee4", "model": "V-Droid (Llama8B)", "organization": "", "configuration": "V-Droid Paper; A11y tree; Training data consists of apps and tasks from the AndroidWorld benchmark. Code.; trials: 1", "score": 0.595, "raw_score": 59.5, "raw_scale": 100, "source_record": "V-Droid (Llama8B)|V-Droid Paper; A11y tree; Training data consists of apps and tasks from the AndroidWorld benchmark. Code.; trials: 1", "source_date": "3/2025" }, { "id": "915a31f2df7cbfbabf67", "model": "Agent S2", "organization": "", "configuration": "Agent S2; Screenshot; trials: 1", "score": 0.5429999999999999, "raw_score": 54.3, "raw_scale": 100, "source_record": "Agent S2|Agent S2; Screenshot; trials: 1", "source_date": "4/2025" }, { "id": "ad4461e9fc8265339bcb", "model": "MAI-UI-2B", "organization": "", "configuration": "MAI-UI; Screenshot; trials: ", "score": 0.491, "raw_score": 49.1, "raw_scale": 100, "source_record": "MAI-UI-2B|MAI-UI; Screenshot; trials: ", "source_date": "1/2026" }, { "id": "8bf86e0aeb50bc63f3cf", "model": "Venus-Navi-7B", "organization": "", "configuration": "UI-Venus; Screenshot; https://huggingface.co/inclusionAI/UI-Venus-Navi-7B; trials: 1", "score": 0.491, "raw_score": 49.1, "raw_scale": 100, "source_record": "Venus-Navi-7B|UI-Venus; Screenshot; https://huggingface.co/inclusionAI/UI-Venus-Navi-7B; trials: 1", "source_date": "8/2025", "report_url": "https://github.com/inclusionAI/UI-Venus/blob/main/vis_androidworld/UI-Venus-androidworld.zip" }, { "id": "44c4700d323daec900bc", "model": "GPT-4o", "organization": "OpenAI", "configuration": "GUI-Explorer; Screenshot + A11y tree; trials: 1", "score": 0.474, "raw_score": 47.4, "raw_scale": 100, "source_record": "GPT-4o|GUI-Explorer; Screenshot + A11y tree; trials: 1", "source_date": "05/2025" }, { "id": "70a4ca183b405582c51c", "model": "GPT-4o", "organization": "OpenAI", "configuration": "AndroidGen; A11y tree; trials: 1", "score": 0.46799999999999997, "raw_score": 46.8, "raw_scale": 100, "source_record": "GPT-4o|AndroidGen; A11y tree; trials: 1", "source_date": "4/2025" }, { "id": "548d347fe63f545bdaaa", "model": "UI-TARS", "organization": "", "configuration": "UI-TARS; Screenshot; trials: 1", "score": 0.466, "raw_score": 46.6, "raw_scale": 100, "source_record": "UI-TARS|UI-TARS; Screenshot; trials: 1", "source_date": "1/2025" }, { "id": "136a50eaa865a7b4275e", "model": "GPT-4o + Aria-UI", "organization": "OpenAI", "configuration": "Aria-UI; Screenshot; trials: 1", "score": 0.44799999999999995, "raw_score": 44.8, "raw_scale": 100, "source_record": "GPT-4o + Aria-UI|Aria-UI; Screenshot; trials: 1", "source_date": "12/2024" }, { "id": "894e7a65f3e6ecbb6adf", "model": "GPT-4o + UGround", "organization": "OpenAI", "configuration": "UGround; Screenshot; Code for reproduction; trials: 1", "score": 0.44, "raw_score": 44.0, "raw_scale": 100, "source_record": "GPT-4o + UGround|UGround; Screenshot; Code for reproduction; trials: 1", "source_date": "1/2025" }, { "id": "1964b362813359f12ac1", "model": "ScaleTrack-7B", "organization": "", "configuration": "ScaleTrack; A11y tree; trials: 1", "score": 0.44, "raw_score": 44.0, "raw_scale": 100, "source_record": "ScaleTrack-7B|ScaleTrack; A11y tree; trials: 1", "source_date": "4/2025" }, { "id": "50951d66e78a43502671", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Mirage-1; Screenshot; With Mirage-1-O; uses OS-Atlas grounder; trials: 1", "score": 0.42200000000000004, "raw_score": 42.2, "raw_scale": 100, "source_record": "GPT-4o|Mirage-1; Screenshot; With Mirage-1-O; uses OS-Atlas grounder; trials: 1", "source_date": "6/2025" }, { "id": "a3d88ce0b5a4fe0391a2", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Ponder & Press; Screenshot; Code is not yet open-sourced; trials: 1", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "GPT-4o|Ponder & Press; Screenshot; Code is not yet open-sourced; trials: 1", "source_date": "12/2024" }, { "id": "763b5e000cff98c9a80f", "model": "GPT-4 Turbo", "organization": "OpenAI", "configuration": "AndroidWorld; A11y tree; trials: 1", "score": 0.306, "raw_score": 30.6, "raw_scale": 100, "source_record": "GPT-4 Turbo|AndroidWorld; A11y tree; trials: 1", "source_date": "05/2024" }, { "id": "f657c79b6bc11a384bff", "model": "GPT-4o, OS-Atlas-Pro 4B, Qwen2-VL-2B-Instruct", "organization": "OpenAI", "configuration": "EcoAgent; Screenshot; trials: 1", "score": 0.276, "raw_score": 27.6, "raw_scale": 100, "source_record": "GPT-4o, OS-Atlas-Pro 4B, Qwen2-VL-2B-Instruct|EcoAgent; Screenshot; trials: 1", "source_date": "05/2024" }, { "id": "282525dcb07f9fc7420b", "model": "Qwen-2.5-VL-7B", "organization": "Alibaba", "configuration": "GUI-Critic-R1; Screenshot + A11y tree; trials: 1", "score": 0.276, "raw_score": 27.6, "raw_scale": 100, "source_record": "Qwen-2.5-VL-7B|GUI-Critic-R1; Screenshot + A11y tree; trials: 1", "source_date": "6/2025" }, { "id": "22c4be3cd9a903d38390", "model": "Qwen2-VL-2B (fine-tuned)", "organization": "Alibaba", "configuration": "InfiGUIAgent; Screenshot; trials: 1", "score": 0.09, "raw_score": 9.0, "raw_scale": 100, "source_record": "Qwen2-VL-2B (fine-tuned)|InfiGUIAgent; Screenshot; trials: 1", "source_date": "1/2025" } ] }, { "id": "bird-sql-test-single", "name": "BIRD-SQL (single-model test)", "group": "编程", "unit": "%", "metric": "Execution accuracy", "protocol": { "en": "Single-model test EX track. Self-consistency settings vary and are retained; Dev, efficiency and Mini-Dev scores are excluded.", "zh": "单模型 Test EX 赛道；保留各结果的自洽采样设置。不混入 Dev、效率分数或 Mini-Dev 成绩。" }, "source": { "data_url": "https://bird-bench.github.io/", "sha256": "9701c60f487025c551f391a0e130434702e7bcfe4385a22a4a1bf9b343cdf03c", "retrieved_at": "2026-11-02T05:57:36.721154+00:00", "name": "BIRD", "url": "https://bird-bench.github.io/", "selector": "Single-model table; Test (%) execution accuracy; not Dev, VES or Mini-Dev", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "a5ecd4877409ceb9bd82", "model": "Gemini-SQL2", "organization": "Google Research & Cloud", "configuration": "Single-model track; self-consistency: Many", "score": 0.8004000000000001, "raw_score": 80.04, "raw_scale": 100, "source_record": "Gemini-SQL2|Single-model track; self-consistency: Many", "source_date": "Jun 3, 2026" }, { "id": "72f32add79a5499aaf14", "model": "ReToolSQL", "organization": "JPMorganChase", "configuration": "Single-model track; self-consistency: Many", "score": 0.7814, "raw_score": 78.14, "raw_scale": 100, "source_record": "ReToolSQL|Single-model track; self-consistency: Many", "source_date": "Sep 29, 2026", "report_url": "https://arxiv.org/pdf/2608.27796" }, { "id": "6ea94e776bf232e18a9b", "model": "SiriusAI-SQL-27B", "organization": "Tencent Data Computing Platform Department", "configuration": "Single-model track; self-consistency: Many", "score": 0.777, "raw_score": 77.7, "raw_scale": 100, "source_record": "SiriusAI-SQL-27B|Single-model track; self-consistency: Many", "source_date": "Oct 10, 2026" }, { "id": "0d2ca8a428208a55f68f", "model": "Gemini-SQL (Multitask SFT + Gemini-2.5-Pro)", "organization": "Google Cloud Databases", "configuration": "Single-model track; self-consistency: Many", "score": 0.7714, "raw_score": 77.14, "raw_scale": 100, "source_record": "Gemini-SQL (Multitask SFT + Gemini-2.5-Pro)|Single-model track; self-consistency: Many", "source_date": "Mar 12, 2026", "report_url": "https://cloud.google.com/blog/products/databases/how-to-get-gemini-to-deeply-understand-your-database" }, { "id": "595ac87c427b6814771d", "model": "SiriusAI-SQL-9B", "organization": "Tencent Data Computing Platform Department", "configuration": "Single-model track; self-consistency: Many", "score": 0.7707999999999999, "raw_score": 77.08, "raw_scale": 100, "source_record": "SiriusAI-SQL-9B|Single-model track; self-consistency: Many", "source_date": "Oct 10, 2026" }, { "id": "fe7be9ecc9e9cc887b19", "model": "Native mini", "organization": "Native", "configuration": "Single-model track; self-consistency: Many", "score": 0.7686, "raw_score": 76.86, "raw_scale": 100, "source_record": "Native mini|Single-model track; self-consistency: Many", "source_date": "Sep 05, 2026" }, { "id": "139d08cccadfc3ddd954", "model": "Q-SQL", "organization": "AWS-Quick Science", "configuration": "Single-model track; self-consistency: Many", "score": 0.7646999999999999, "raw_score": 76.47, "raw_scale": 100, "source_record": "Q-SQL|Single-model track; self-consistency: Many", "source_date": "Dec 4, 2025", "report_url": "https://www.notion.so/BIRD-v2-Memory-2bfe8bf30514801ba112f92ff20784e3" }, { "id": "992d0f25601b99b88750", "model": "Databricks RLVR 32B", "organization": "Databricks Mosaic Research", "configuration": "Single-model track; self-consistency: Few", "score": 0.7568, "raw_score": 75.68, "raw_scale": 100, "source_record": "Databricks RLVR 32B|Single-model track; self-consistency: Few", "source_date": "July 08, 2025", "report_url": "https://www.databricks.com/research" }, { "id": "e788848132b1d7c9067c", "model": "Spektr-SQL", "organization": "Amazon Ads-SpektrBot", "configuration": "Single-model track; self-consistency: Few", "score": 0.7484999999999999, "raw_score": 74.85, "raw_scale": 100, "source_record": "Spektr-SQL|Single-model track; self-consistency: Few", "source_date": "Oct 30, 2025", "report_url": "https://www.notion.so/BIRD-Submission-SQL-291e8bf3051480afa7b8cfac3aa26b10" }, { "id": "07e98ccff8eca927f3aa", "model": "Sophon-Text2SQL-32B", "organization": "ByteDance DataPlatform-LLM", "configuration": "Single-model track; self-consistency: Many", "score": 0.7479, "raw_score": 74.79, "raw_scale": 100, "source_record": "Sophon-Text2SQL-32B|Single-model track; self-consistency: Many", "source_date": "July 22, 2025" }, { "id": "a5460922c8a51b168a59", "model": "ELG-SQL", "organization": "AMS CIR", "configuration": "Single-model track; self-consistency: Many", "score": 0.7451000000000001, "raw_score": 74.51, "raw_scale": 100, "source_record": "ELG-SQL|Single-model track; self-consistency: Many", "source_date": "Sep 02, 2026" }, { "id": "8d8173347e5099df2f7e", "model": "Velldepth-ZenPivot-27B", "organization": "Alibaba Security", "configuration": "Single-model track; self-consistency: Many", "score": 0.7429000000000001, "raw_score": 74.29, "raw_scale": 100, "source_record": "Velldepth-ZenPivot-27B|Single-model track; self-consistency: Many", "source_date": "Sep 28, 2026" }, { "id": "b80ea0c38b261346105b", "model": "Kwai-AutoSQL-32B", "organization": "Kuaishou Business Team", "configuration": "Single-model track; self-consistency: Many", "score": 0.7412000000000001, "raw_score": 74.12, "raw_scale": 100, "source_record": "Kwai-AutoSQL-32B|Single-model track; self-consistency: Many", "source_date": "Sep 19, 2026" }, { "id": "d3ee05a4e8637e04878f", "model": "Kwai-AutoSQL-14B", "organization": "Kuaishou Business Team", "configuration": "Single-model track; self-consistency: Many", "score": 0.7401000000000001, "raw_score": 74.01, "raw_scale": 100, "source_record": "Kwai-AutoSQL-14B|Single-model track; self-consistency: Many", "source_date": "Sep 19, 2026" }, { "id": "a8a1b00a225a39974b3b", "model": "Arctic-Text2SQL-R1-32B", "organization": "Snowflake", "configuration": "Single-model track; self-consistency: Few", "score": 0.7384000000000001, "raw_score": 73.84, "raw_scale": 100, "source_record": "Arctic-Text2SQL-R1-32B|Single-model track; self-consistency: Few", "source_date": "May 07, 2025", "report_url": "https://www.snowflake.com/en/product/ai/ai-research/" }, { "id": "816eaedeee43b101b52e", "model": "flash-sqlite: Qwen3.5-2B Generator + Self-Judge", "organization": "Alio Leuchtmann", "configuration": "Single-model track; self-consistency: Many", "score": 0.7373000000000001, "raw_score": 73.73, "raw_scale": 100, "source_record": "flash-sqlite: Qwen3.5-2B Generator + Self-Judge|Single-model track; self-consistency: Many", "source_date": "Sep 29, 2026" }, { "id": "ad46ee8cead7a84d6d95", "model": "Databricks RLVR 32B", "organization": "Databricks Mosaic Research", "configuration": "Single-model track; self-consistency: ", "score": 0.7356, "raw_score": 73.56, "raw_scale": 100, "source_record": "Databricks RLVR 32B|Single-model track; self-consistency: ", "source_date": "July 08, 2025", "report_url": "https://www.databricks.com/research" }, { "id": "0ea82bebb3649c19bada", "model": "Jiayin-Pangu-Text2SQL-14B", "organization": "Jiayin AI Research Team", "configuration": "Single-model track; self-consistency: Many", "score": 0.7345, "raw_score": 73.45, "raw_scale": 100, "source_record": "Jiayin-Pangu-Text2SQL-14B|Single-model track; self-consistency: Many", "source_date": "Sep 19, 2025" }, { "id": "4ba1755c4f85152dfdb3", "model": "FTEL-Text2SQL", "organization": "FPT Telecom", "configuration": "Single-model track; self-consistency: Few", "score": 0.7278, "raw_score": 72.78, "raw_scale": 100, "source_record": "FTEL-Text2SQL|Single-model track; self-consistency: Few", "source_date": "May 4, 2026" }, { "id": "717d2d97ecbf6c258eff", "model": "GPT-5.5-xhigh", "organization": "OpenAI", "configuration": "Single-model track; self-consistency: ", "score": 0.7254999999999999, "raw_score": 72.55, "raw_scale": 100, "source_record": "GPT-5.5-xhigh|Single-model track; self-consistency: ", "source_date": "Apr 28, 2026" }, { "id": "168e23edb1d8de22c899", "model": "Arctic-Text2SQL-R1-14B", "organization": "Snowflake", "configuration": "Single-model track; self-consistency: Few", "score": 0.7222, "raw_score": 72.22, "raw_scale": 100, "source_record": "Arctic-Text2SQL-R1-14B|Single-model track; self-consistency: Few", "source_date": "May 07, 2025", "report_url": "https://www.snowflake.com/en/product/ai/ai-research/" }, { "id": "3ce957cccdb25b97be4a", "model": "Fractal-SQL", "organization": "Fractal Analytics", "configuration": "Single-model track; self-consistency: Many", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "Fractal-SQL|Single-model track; self-consistency: Many", "source_date": "Oct 01, 2025", "report_url": "https://fractal.ai/" }, { "id": "03a247bf24eae14354e9", "model": "SQLWeaver-32B", "organization": "Alibaba AI Data", "configuration": "Single-model track; self-consistency: ", "score": 0.7166, "raw_score": 71.66, "raw_scale": 100, "source_record": "SQLWeaver-32B|Single-model track; self-consistency: ", "source_date": "May 15, 2026" }, { "id": "5dd4dd9bbde1822b4d0a", "model": "ShepherdSQL", "organization": "JD Bitcircuit Research", "configuration": "Single-model track; self-consistency: Many", "score": 0.7116, "raw_score": 71.16, "raw_scale": 100, "source_record": "ShepherdSQL|Single-model track; self-consistency: Many", "source_date": "Sep 15, 2025" }, { "id": "7e65e2a68c78e184985c", "model": "JD-UNION-Text2sql-32B", "organization": "JD UNION", "configuration": "Single-model track; self-consistency: Many", "score": 0.7093, "raw_score": 70.93, "raw_scale": 100, "source_record": "JD-UNION-Text2sql-32B|Single-model track; self-consistency: Many", "source_date": "Jan 13, 2026" }, { "id": "6855b3f4162ac7687074", "model": "SIFT-32B", "organization": "Seek AI", "configuration": "Single-model track; self-consistency: Scale", "score": 0.7093, "raw_score": 70.93, "raw_scale": 100, "source_record": "SIFT-32B|Single-model track; self-consistency: Scale", "source_date": "May 27, 2025", "report_url": "https://www.seek.ai/research" }, { "id": "b6425c5ad88f2eae978a", "model": "CrazyData-Text2SQL-32B", "organization": "JD Bitcircuit Research", "configuration": "Single-model track; self-consistency: Many", "score": 0.7071, "raw_score": 70.71, "raw_scale": 100, "source_record": "CrazyData-Text2SQL-32B|Single-model track; self-consistency: Many", "source_date": "Sep 15, 2025" }, { "id": "b593c89fc5ea5b78639d", "model": "Infly-RL-SQL-32B", "organization": "inftech.ai", "configuration": "Single-model track; self-consistency: Few", "score": 0.706, "raw_score": 70.6, "raw_scale": 100, "source_record": "Infly-RL-SQL-32B|Single-model track; self-consistency: Few", "source_date": "May 08, 2025" }, { "id": "90ee1afb27853838a403", "model": "Arctic-Text2SQL-R1-7B", "organization": "Snowflake", "configuration": "Single-model track; self-consistency: Few", "score": 0.7043, "raw_score": 70.43, "raw_scale": 100, "source_record": "Arctic-Text2SQL-R1-7B|Single-model track; self-consistency: Few", "source_date": "May 07, 2025", "report_url": "https://www.snowflake.com/en/product/ai/ai-research/" }, { "id": "f3553659b62926c4357c", "model": "Claude Opus 4.6", "organization": "Anthropic", "configuration": "Single-model track; self-consistency: ", "score": 0.7015, "raw_score": 70.15, "raw_scale": 100, "source_record": "Claude Opus 4.6|Single-model track; self-consistency: ", "source_date": "Feb 21, 2026" }, { "id": "f024918c6c43d94fc694", "model": "XiYanSQL-QwenCoder-32B", "organization": "Alibaba Cloud", "configuration": "Single-model track; self-consistency: Few", "score": 0.6903, "raw_score": 69.03, "raw_scale": 100, "source_record": "XiYanSQL-QwenCoder-32B|Single-model track; self-consistency: Few", "source_date": "Jan 09, 2025", "report_url": "https://arxiv.org/abs/2507.04701" }, { "id": "0fb0dc8f429074a59275", "model": "Arctic-ExCoT-70B", "organization": "Snowflake", "configuration": "Single-model track; self-consistency: ", "score": 0.6853, "raw_score": 68.53, "raw_scale": 100, "source_record": "Arctic-ExCoT-70B|Single-model track; self-consistency: ", "source_date": "Mar 12, 2025", "report_url": "https://arxiv.org/abs/2503.19988" }, { "id": "b5ed5cae19c033c6ff6e", "model": "Qwen3.5-2B + StructCoT Distillation", "organization": "Alio Leuchtmann", "configuration": "Single-model track; self-consistency: Few", "score": 0.6847, "raw_score": 68.47, "raw_scale": 100, "source_record": "Qwen3.5-2B + StructCoT Distillation|Single-model track; self-consistency: Few", "source_date": "Aug 24, 2026" }, { "id": "d4b09a7ae6633c020a35", "model": "Arctic-ExCoT-32B", "organization": "Snowflake", "configuration": "Single-model track; self-consistency: ", "score": 0.6819, "raw_score": 68.19, "raw_scale": 100, "source_record": "Arctic-ExCoT-32B|Single-model track; self-consistency: ", "source_date": "Mar 12, 2025", "report_url": "https://arxiv.org/abs/2503.19988" }, { "id": "492157388aafbf7e0a8c", "model": "Qwen3-Coder-480B-A35B", "organization": "Alibaba", "configuration": "Single-model track; self-consistency: ", "score": 0.6814, "raw_score": 68.14, "raw_scale": 100, "source_record": "Qwen3-Coder-480B-A35B|Single-model track; self-consistency: ", "source_date": "Feb 21, 2026" }, { "id": "4bcab11b10e8c3cb0bbb", "model": "Align-SQL-7B", "organization": "Anonymous", "configuration": "Single-model track; self-consistency: Few", "score": 0.6797, "raw_score": 67.97, "raw_scale": 100, "source_record": "Align-SQL-7B|Single-model track; self-consistency: Few", "source_date": "Aug 09, 2026" }, { "id": "d4f9fcde54619a17bf14", "model": "Claude 4.5 Sonnet", "organization": "Anthropic", "configuration": "Single-model track; self-consistency: ", "score": 0.6685, "raw_score": 66.85, "raw_scale": 100, "source_record": "Claude 4.5 Sonnet|Single-model track; self-consistency: ", "source_date": "Feb 21, 2026" }, { "id": "a864c69d306a41452bcb", "model": "Align-SQL-3B", "organization": "Anonymous", "configuration": "Single-model track; self-consistency: Few", "score": 0.6640999999999999, "raw_score": 66.41, "raw_scale": 100, "source_record": "Align-SQL-3B|Single-model track; self-consistency: Few", "source_date": "Aug 09, 2026" }, { "id": "37e7a12479028ee981f2", "model": "Command A", "organization": "Cohere AI", "configuration": "Single-model track; self-consistency: ", "score": 0.6568, "raw_score": 65.68, "raw_scale": 100, "source_record": "Command A|Single-model track; self-consistency: ", "source_date": "Apr 30, 2025", "report_url": "https://arxiv.org/abs/2504.00698" }, { "id": "12ba8bd201bb4361795b", "model": "OneSQL-v0.1-Qwen-32B", "organization": "onekq.ai", "configuration": "Single-model track; self-consistency: ", "score": 0.6333, "raw_score": 63.33, "raw_scale": 100, "source_record": "OneSQL-v0.1-Qwen-32B|Single-model track; self-consistency: ", "source_date": "Mar 11, 2025" }, { "id": "7ed12c2c830e66b72240", "model": "GLM-4.7", "organization": "Z AI", "configuration": "Single-model track; self-consistency: ", "score": 0.6294, "raw_score": 62.94, "raw_scale": 100, "source_record": "GLM-4.7|Single-model track; self-consistency: ", "source_date": "Feb 21, 2026" }, { "id": "f47226adff0c1cb977da", "model": "SuperSQL", "organization": "HKUST(GZ)", "configuration": "Single-model track; self-consistency: ", "score": 0.6265999999999999, "raw_score": 62.66, "raw_scale": 100, "source_record": "SuperSQL|Single-model track; self-consistency: ", "source_date": "Jun 1, 2024", "report_url": "https://arxiv.org/abs/2406.01265" }, { "id": "c879ed1d12832225d591", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Single-model track; self-consistency: ", "score": 0.6093, "raw_score": 60.93, "raw_scale": 100, "source_record": "DeepSeek-R1|Single-model track; self-consistency: ", "source_date": "Feb 21, 2026" }, { "id": "8c0815a2d8f60b364511", "model": "Struct-SQL", "organization": "Crater Labs", "configuration": "Single-model track; self-consistency: ", "score": 0.6042000000000001, "raw_score": 60.42, "raw_scale": 100, "source_record": "Struct-SQL|Single-model track; self-consistency: ", "source_date": "Jan 30, 2026", "report_url": "https://arxiv.org/abs/2512.17053" }, { "id": "18fdd596061face5e5f7", "model": "SFT CodeS-15B", "organization": "Renmin University of China", "configuration": "Single-model track; self-consistency: ", "score": 0.6037, "raw_score": 60.37, "raw_scale": 100, "source_record": "SFT CodeS-15B|Single-model track; self-consistency: ", "source_date": "Oct 12, 2023", "report_url": "https://arxiv.org/abs/2402.16347" }, { "id": "fdbdc53040dd14695071", "model": "Kimi-K2-Thinking", "organization": "Moonshot AI", "configuration": "Single-model track; self-consistency: ", "score": 0.5987, "raw_score": 59.87, "raw_scale": 100, "source_record": "Kimi-K2-Thinking|Single-model track; self-consistency: ", "source_date": "Feb 21, 2026" }, { "id": "6fe29c1695d1777b4847", "model": "xorazm-text2sql-0.8b", "organization": "Martin Luther University Halle-Wittenberg", "configuration": "Single-model track; self-consistency: Few", "score": 0.5959, "raw_score": 59.59, "raw_scale": 100, "source_record": "xorazm-text2sql-0.8b|Single-model track; self-consistency: Few", "source_date": "Jul 17, 2026", "report_url": "https://huggingface.co/shakha-de/xorazm-text2sql-0.8b" }, { "id": "e30d4ed34f3565e0a5e7", "model": "SFT CodeS-7B", "organization": "Renmin University of China", "configuration": "Single-model track; self-consistency: ", "score": 0.5925, "raw_score": 59.25, "raw_scale": 100, "source_record": "SFT CodeS-7B|Single-model track; self-consistency: ", "source_date": "Oct 12, 2023", "report_url": "https://arxiv.org/abs/2402.16347" }, { "id": "146eca956ca4d23057e7", "model": "DeepSeek", "organization": "DeepSeek", "configuration": "Single-model track; self-consistency: ", "score": 0.5668, "raw_score": 56.68, "raw_scale": 100, "source_record": "DeepSeek|Single-model track; self-consistency: ", "source_date": "Aug 10, 2024" }, { "id": "7e4b49d360fc674cb0b8", "model": "Mistral", "organization": "Mistral AI", "configuration": "Single-model track; self-consistency: ", "score": 0.5584, "raw_score": 55.84, "raw_scale": 100, "source_record": "Mistral|Single-model track; self-consistency: ", "source_date": "Aug 08, 2024" }, { "id": "ef1b3ff7105da43d00a9", "model": "GPT-4", "organization": "OpenAI", "configuration": "Single-model track; self-consistency: ", "score": 0.5489, "raw_score": 54.89, "raw_scale": 100, "source_record": "GPT-4|Single-model track; self-consistency: ", "source_date": "Jul 01, 2023" }, { "id": "589ebca89cbeda4f32a8", "model": "Prem-1B-SQL", "organization": "Prem AI", "configuration": "Single-model track; self-consistency: Few", "score": 0.5154, "raw_score": 51.54, "raw_scale": 100, "source_record": "Prem-1B-SQL|Single-model track; self-consistency: Few", "source_date": "Sep 19, 2024" }, { "id": "4bd2eb4ef45d8e2bf267", "model": "Claude-2", "organization": "Anthropic", "configuration": "Single-model track; self-consistency: ", "score": 0.4902, "raw_score": 49.02, "raw_scale": 100, "source_record": "Claude-2|Single-model track; self-consistency: ", "source_date": "Jul 16, 2023" }, { "id": "3ddcfc85d20b28265e7f", "model": "ChatGPT + CoT", "organization": "HKU & DAMO", "configuration": "Single-model track; self-consistency: ", "score": 0.4008, "raw_score": 40.08, "raw_scale": 100, "source_record": "ChatGPT + CoT|Single-model track; self-consistency: ", "source_date": "Mar 17, 2023", "report_url": "https://arxiv.org/abs/2305.03111" }, { "id": "acc56a4f470946eea9bc", "model": "ChatGPT", "organization": "OpenAI", "configuration": "Single-model track; self-consistency: ", "score": 0.39299999999999996, "raw_score": 39.3, "raw_scale": 100, "source_record": "ChatGPT|Single-model track; self-consistency: ", "source_date": "Mar 17, 2023" }, { "id": "f981d16dc32b5bb52f2a", "model": "Codex", "organization": "", "configuration": "Single-model track; self-consistency: ", "score": 0.36469999999999997, "raw_score": 36.47, "raw_scale": 100, "source_record": "Codex|Single-model track; self-consistency: ", "source_date": "Feb 17, 2023" }, { "id": "e7fb5170b5824f3ab205", "model": "sqlpup (7-sample vote)", "organization": "", "configuration": "Single-model track; self-consistency: Few", "score": 0.33149999999999996, "raw_score": 33.15, "raw_scale": 100, "source_record": "sqlpup (7-sample vote)|Single-model track; self-consistency: Few", "source_date": "Aug 24, 2026", "report_url": "https://github.com/shivenkk/sqlpup" }, { "id": "c5866a9ddcfb2e05ddb0", "model": "Palm-2", "organization": "Google", "configuration": "Single-model track; self-consistency: ", "score": 0.33039999999999997, "raw_score": 33.04, "raw_scale": 100, "source_record": "Palm-2|Single-model track; self-consistency: ", "source_date": "Jul 16, 2023" }, { "id": "b99849d5d75e20978ef0", "model": "sqlpup (single pass)", "organization": "", "configuration": "Single-model track; self-consistency: ", "score": 0.2733, "raw_score": 27.33, "raw_scale": 100, "source_record": "sqlpup (single pass)|Single-model track; self-consistency: ", "source_date": "Aug 24, 2026", "report_url": "https://github.com/shivenkk/sqlpup" }, { "id": "7329fd7628036bbef1e3", "model": "T5-3B", "organization": "", "configuration": "Single-model track; self-consistency: ", "score": 0.24050000000000002, "raw_score": 24.05, "raw_scale": 100, "source_record": "T5-3B|Single-model track; self-consistency: ", "source_date": "Feb 5, 2023" }, { "id": "e2b3e8dad7a85a4aa63e", "model": "T5-Large", "organization": "", "configuration": "Single-model track; self-consistency: ", "score": 0.2094, "raw_score": 20.94, "raw_scale": 100, "source_record": "T5-Large|Single-model track; self-consistency: ", "source_date": "Feb 3, 2023" }, { "id": "fc88809f65f7c731eb0f", "model": "T5-Base", "organization": "", "configuration": "Single-model track; self-consistency: ", "score": 0.12890000000000001, "raw_score": 12.89, "raw_scale": 100, "source_record": "T5-Base|Single-model track; self-consistency: ", "source_date": "Feb 3, 2023" } ] }, { "id": "spider-2-lite", "name": "Spider 2.0-Lite", "group": "编程", "unit": "%", "metric": "Task success", "protocol": { "en": "Official Lite submission track, 547 tasks. Model and agent names follow the authors’ table.", "zh": "官方 Lite 提交赛道，共 547 道任务。模型与 Agent 名称取自作者的原表。" }, "source": { "data_url": "https://spider2-sql.github.io/", "sha256": "3cd4bb95ed0f2d9d5e3aa76cb2461f9f22c8af8ac7b7d6136e3589ce01474e61", "retrieved_at": "2026-11-02T05:57:36.441642+00:00", "name": "Spider 2.0", "url": "https://spider2-sql.github.io/", "selector": "#ex-table-lite Score / 100; latest dated identical submission", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "f0880cc726d064008567", "model": "Tianqiong Data Agent + GLM 5.2", "organization": "Data Computing Platform Department, Tencent", "configuration": "Spider 2.0-Lite", "score": 0.7623000000000001, "raw_score": 76.23, "raw_scale": 100, "source_record": "Tianqiong Data Agent + GLM 5.2|Spider 2.0-Lite", "source_date": "Jul 28, 2026" }, { "id": "289c43756f0f389d886e", "model": "DecisionX Agent", "organization": "DecisionX AI", "configuration": "Spider 2.0-Lite", "score": 0.7495, "raw_score": 74.95, "raw_scale": 100, "source_record": "DecisionX Agent|Spider 2.0-Lite", "source_date": "Jul 28, 2026", "report_url": "https://www.decisionx.ai/" }, { "id": "17fc9511eefb8725f7fc", "model": "ktx", "organization": "Kaelio", "configuration": "Spider 2.0-Lite", "score": 0.7367, "raw_score": 73.67, "raw_scale": 100, "source_record": "ktx|Spider 2.0-Lite", "source_date": "Jun 30, 2026", "report_url": "https://github.com/Kaelio/ktx" }, { "id": "d7483c7fdb4a99ebf39f", "model": "DivSkill-SQL", "organization": "Snowflake AI Research x UCSD", "configuration": "Spider 2.0-Lite", "score": 0.7313, "raw_score": 73.13, "raw_scale": 100, "source_record": "DivSkill-SQL|Spider 2.0-Lite", "source_date": "May 22, 2026" }, { "id": "d8371a3f415384c30501", "model": "SOMA-SQL", "organization": "Oracle Cloud Infrastructure, AI Science team", "configuration": "Spider 2.0-Lite", "score": 0.7202, "raw_score": 72.02, "raw_scale": 100, "source_record": "SOMA-SQL|Spider 2.0-Lite", "source_date": "Apr 11, 2026", "report_url": "https://arxiv.org/abs/2606.11424" }, { "id": "a70516bfc7317254fecb", "model": "Databao Agent", "organization": "Databao by JetBrains", "configuration": "Spider 2.0-Lite", "score": 0.6965, "raw_score": 69.65, "raw_scale": 100, "source_record": "Databao Agent|Spider 2.0-Lite", "source_date": "Mar 10, 2026", "report_url": "https://databao.app/" }, { "id": "607460764211d4042b6c", "model": "QUVI-2.3 + Claude-Opus-4.5", "organization": "DAQUV", "configuration": "Spider 2.0-Lite", "score": 0.6581, "raw_score": 65.81, "raw_scale": 100, "source_record": "QUVI-2.3 + Claude-Opus-4.5|Spider 2.0-Lite", "source_date": "Jan 12, 2025", "report_url": "https://daquv.com/en" }, { "id": "3f935ce20e3ad7002c5b", "model": "EXA-SQL", "organization": "MJ Park x MS Sim", "configuration": "Spider 2.0-Lite", "score": 0.6416, "raw_score": 64.16, "raw_scale": 100, "source_record": "EXA-SQL|Spider 2.0-Lite", "source_date": "Nov 29, 2025" }, { "id": "1091b5362ff89a1425d5", "model": "nQuery", "organization": "nGenux", "configuration": "Spider 2.0-Lite", "score": 0.619, "raw_score": 61.9, "raw_scale": 100, "source_record": "nQuery|Spider 2.0-Lite", "source_date": "May 15, 2026", "report_url": "https://www.ngenux.com/" }, { "id": "310d011cd6108ae73cda", "model": "Claude Code Agent + Sonnet", "organization": "Nodal Data", "configuration": "Spider 2.0-Lite", "score": 0.612, "raw_score": 61.2, "raw_scale": 100, "source_record": "Claude Code Agent + Sonnet|Spider 2.0-Lite", "source_date": "May 7, 2026" }, { "id": "6531e2095abd68a8cf07", "model": "ProSPy: Profiling-driven SQL-Python Analysis Framework + Claude-4.5-Opus", "organization": "Data Computing Platform Department, Tencent", "configuration": "Spider 2.0-Lite", "score": 0.6015, "raw_score": 60.15, "raw_scale": 100, "source_record": "ProSPy: Profiling-driven SQL-Python Analysis Framework + Claude-4.5-Opus|Spider 2.0-Lite", "source_date": "Feb 13, 2026" }, { "id": "238707e7b5ee3045f6a9", "model": "ReFoRCE + o3", "organization": "Hao AI Lab x Snowflake", "configuration": "Spider 2.0-Lite", "score": 0.5521, "raw_score": 55.21, "raw_scale": 100, "source_record": "ReFoRCE + o3|Spider 2.0-Lite", "source_date": "Nov 3, 2025", "report_url": "https://arxiv.org/abs/2502.00675" }, { "id": "29339b5b17327716d865", "model": "CoFD-SQL + GPT-5", "organization": "Samsung SDS Research America", "configuration": "Spider 2.0-Lite", "score": 0.5466, "raw_score": 54.66, "raw_scale": 100, "source_record": "CoFD-SQL + GPT-5|Spider 2.0-Lite", "source_date": "Nov 1, 2025" }, { "id": "d2fd2d5930b68e1f8e12", "model": "CT-ChatBI + o4-mini", "organization": "CT-ChatBI Team", "configuration": "Spider 2.0-Lite", "score": 0.5302, "raw_score": 53.02, "raw_scale": 100, "source_record": "CT-ChatBI + o4-mini|Spider 2.0-Lite", "source_date": "May 20, 2026" }, { "id": "5cd59a3b5ea5c70f13de", "model": "AutoLink + Deepseek-R1", "organization": "HUST VLR Lab", "configuration": "Spider 2.0-Lite", "score": 0.5228, "raw_score": 52.28, "raw_scale": 100, "source_record": "AutoLink + Deepseek-R1|Spider 2.0-Lite", "source_date": "Jul 19, 2025", "report_url": "https://arxiv.org/pdf/2511.17190" }, { "id": "84d6836fb41aa44f3336", "model": "GRASP + Reforce + Gemini-3-Flash + DeepSeek-V3.2", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.5131, "raw_score": 51.31, "raw_scale": 100, "source_record": "GRASP + Reforce + Gemini-3-Flash + DeepSeek-V3.2|Spider 2.0-Lite", "source_date": "Feb 27, 2026" }, { "id": "2dfac914f50f19274212", "model": "AgenticData + Qwen3", "organization": "Tsinghua University", "configuration": "Spider 2.0-Lite", "score": 0.445, "raw_score": 44.5, "raw_scale": 100, "source_record": "AgenticData + Qwen3|Spider 2.0-Lite", "source_date": "Aug 7, 2025", "report_url": "https://arxiv.org/abs/2508.05002" }, { "id": "ce4c71c152cb010c5d08", "model": "AI-DIVE + DeepSeek-V4-Flash", "organization": "S-Core", "configuration": "Spider 2.0-Lite", "score": 0.424, "raw_score": 42.4, "raw_scale": 100, "source_record": "AI-DIVE + DeepSeek-V4-Flash|Spider 2.0-Lite", "source_date": "Jul 7, 2026", "report_url": "https://s-core.co.kr/en/" }, { "id": "34c6b5a5b7fe16b6ffdf", "model": "Spider-Agent + Claude-Sonnet-4.5", "organization": "Myoseop Sim", "configuration": "Spider 2.0-Lite", "score": 0.41859999999999997, "raw_score": 41.86, "raw_scale": 100, "source_record": "Spider-Agent + Claude-Sonnet-4.5|Spider 2.0-Lite", "source_date": "Nov 10, 2025" }, { "id": "fd471db5a2daeaead51b", "model": "Dr.Q", "organization": "Dr.Q Lab", "configuration": "Spider 2.0-Lite", "score": 0.35600000000000004, "raw_score": 35.6, "raw_scale": 100, "source_record": "Dr.Q|Spider 2.0-Lite", "source_date": "Jul 14, 2026" }, { "id": "6910a3a78ea4b0a325ac", "model": "ReFoRCE + Qwen3", "organization": "Tsinghua University", "configuration": "Spider 2.0-Lite", "score": 0.35600000000000004, "raw_score": 35.6, "raw_scale": 100, "source_record": "ReFoRCE + Qwen3|Spider 2.0-Lite", "source_date": "Aug 7, 2025", "report_url": "https://arxiv.org/abs/2508.05002" }, { "id": "fd18143770d143ad3eb2", "model": "LinkAlign + DeepSeek-R1", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.3309, "raw_score": 33.09, "raw_scale": 100, "source_record": "LinkAlign + DeepSeek-R1|Spider 2.0-Lite", "source_date": "Apr 27, 2025", "report_url": "https://arxiv.org/abs/2503.18596" }, { "id": "4265b611fcc81482a3fe", "model": "RSL-SQL + o3", "organization": "HUST VLR Lab", "configuration": "Spider 2.0-Lite", "score": 0.3309, "raw_score": 33.09, "raw_scale": 100, "source_record": "RSL-SQL + o3|Spider 2.0-Lite", "source_date": "Jul 10, 2025", "report_url": "https://arxiv.org/abs/2411.00073" }, { "id": "4299db43b73e52cc8d22", "model": "YAN-Agent + GLM-5", "organization": "Kuaishou", "configuration": "Spider 2.0-Lite", "score": 0.3236, "raw_score": 32.36, "raw_scale": 100, "source_record": "YAN-Agent + GLM-5|Spider 2.0-Lite", "source_date": "May 21, 2026" }, { "id": "05f6adc95e8a1dea32f7", "model": "RSL-SQL + DeepSeek-R1", "organization": "HUST VLR Lab", "configuration": "Spider 2.0-Lite", "score": 0.3053, "raw_score": 30.53, "raw_scale": 100, "source_record": "RSL-SQL + DeepSeek-R1|Spider 2.0-Lite", "source_date": "Jul 10, 2025", "report_url": "https://arxiv.org/abs/2411.00073" }, { "id": "ca9321baac7400fc3f18", "model": "ReFoRCE + o1-preview", "organization": "Hao AI Lab x Snowflake", "configuration": "Spider 2.0-Lite", "score": 0.3035, "raw_score": 30.35, "raw_scale": 100, "source_record": "ReFoRCE + o1-preview|Spider 2.0-Lite", "source_date": "Jan 28, 2025", "report_url": "https://arxiv.org/abs/2502.00675" }, { "id": "87aab70a7b75bdc45bee", "model": "Spider-Agent + Claude-3.7-Sonnet-20250219-Thinking", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.2852, "raw_score": 28.52, "raw_scale": 100, "source_record": "Spider-Agent + Claude-3.7-Sonnet-20250219-Thinking|Spider 2.0-Lite", "source_date": "Mar 16, 2025" }, { "id": "635b0dc518dd02e4c3c8", "model": "Spider-Agent + Claude-4-Sonnet-20250514", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.2779, "raw_score": 27.79, "raw_scale": 100, "source_record": "Spider-Agent + Claude-4-Sonnet-20250514|Spider 2.0-Lite", "source_date": "Mar 31, 2025" }, { "id": "28d5ef218a9557323bf3", "model": "RSL-SQL + DeepSeek-V3", "organization": "HUST VLR Lab", "configuration": "Spider 2.0-Lite", "score": 0.2614, "raw_score": 26.14, "raw_scale": 100, "source_record": "RSL-SQL + DeepSeek-V3|Spider 2.0-Lite", "source_date": "Jul 10, 2025", "report_url": "https://arxiv.org/abs/2411.00073" }, { "id": "7e9efdf85819168a96a9", "model": "Spider-Agent + Claude-3.7-Sonnet-20250219", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.2541, "raw_score": 25.41, "raw_scale": 100, "source_record": "Spider-Agent + Claude-3.7-Sonnet-20250219|Spider 2.0-Lite", "source_date": "Mar 8, 2025" }, { "id": "598c63a679b44396507c", "model": "LinkAlign + DeepSeek-V3", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.2486, "raw_score": 24.86, "raw_scale": 100, "source_record": "LinkAlign + DeepSeek-V3|Spider 2.0-Lite", "source_date": "Mar 28, 2025", "report_url": "https://arxiv.org/abs/2503.18596" }, { "id": "cbf79cd1a02fe4b5e129", "model": "Spider-Agent + o3-mini-2025-01-31", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.23399999999999999, "raw_score": 23.4, "raw_scale": 100, "source_record": "Spider-Agent + o3-mini-2025-01-31|Spider 2.0-Lite", "source_date": "Feb 10, 2025" }, { "id": "dda4c66a79cc520e443c", "model": "Spider-Agent + o1-preview", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.2303, "raw_score": 23.03, "raw_scale": 100, "source_record": "Spider-Agent + o1-preview|Spider 2.0-Lite", "source_date": "Nov 30, 2024" }, { "id": "22b34ff543a55d00826f", "model": "AI-DIVE + gpt-oss-120B", "organization": "S-Core", "configuration": "Spider 2.0-Lite", "score": 0.21899999999999997, "raw_score": 21.9, "raw_scale": 100, "source_record": "AI-DIVE + gpt-oss-120B|Spider 2.0-Lite", "source_date": "Apr 20, 2026", "report_url": "https://s-core.co.kr/en/" }, { "id": "43a4d2fe8fc0cbe03a72", "model": "Spider-Agent + DeepSeek-R1", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.1371, "raw_score": 13.71, "raw_scale": 100, "source_record": "Spider-Agent + DeepSeek-R1|Spider 2.0-Lite", "source_date": "Mar 10, 2025" }, { "id": "e0f8a5777d2587e1326e", "model": "Spider-Agent + GPT-4o-2024-11-20", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.1316, "raw_score": 13.16, "raw_scale": 100, "source_record": "Spider-Agent + GPT-4o-2024-11-20|Spider 2.0-Lite", "source_date": "Feb 10, 2025" }, { "id": "695f5f3ae6110401df6f", "model": "Spider-Agent + QwQ-32B", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.1133, "raw_score": 11.33, "raw_scale": 100, "source_record": "Spider-Agent + QwQ-32B|Spider 2.0-Lite", "source_date": "Mar 13, 2025" }, { "id": "ffd36af36727a172c1f3", "model": "Duo", "organization": "Anonymous", "configuration": "Spider 2.0-Lite", "score": 0.08960000000000001, "raw_score": 8.96, "raw_scale": 100, "source_record": "Duo|Spider 2.0-Lite", "source_date": "Dec 31, 2024" }, { "id": "bdff422b898d406d1162", "model": "Spider-Agent + Claude-3.5-Sonnet-20240620", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.0832, "raw_score": 8.32, "raw_scale": 100, "source_record": "Spider-Agent + Claude-3.5-Sonnet-20240620|Spider 2.0-Lite", "source_date": "Nov 2, 2024" }, { "id": "d7b052531b4d29c0cea3", "model": "Spider-Agent + Qwen2.5-Coder-32B-Instruct", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.058499999999999996, "raw_score": 5.85, "raw_scale": 100, "source_record": "Spider-Agent + Qwen2.5-Coder-32B-Instruct|Spider 2.0-Lite", "source_date": "Nov 2, 2024" }, { "id": "d852e2df8e54782920ec", "model": "DailSQL + GPT-4o", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.056799999999999996, "raw_score": 5.68, "raw_scale": 100, "source_record": "DailSQL + GPT-4o|Spider 2.0-Lite", "source_date": "Nov 2, 2024" }, { "id": "df6f499729ba1379ce7e", "model": "CHESS + GPT-4o", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.0384, "raw_score": 3.84, "raw_scale": 100, "source_record": "CHESS + GPT-4o|Spider 2.0-Lite", "source_date": "Nov 2, 2024" }, { "id": "3d51e5f8d30d2984fd4d", "model": "DIN-SQL + GPT-4o", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.0146, "raw_score": 1.46, "raw_scale": 100, "source_record": "DIN-SQL + GPT-4o|Spider 2.0-Lite", "source_date": "Nov 2, 2024" }, { "id": "d7a36473f9160bdf8b8f", "model": "SFT CodeS-15B", "organization": "", "configuration": "Spider 2.0-Lite", "score": 0.0073, "raw_score": 0.73, "raw_scale": 100, "source_record": "SFT CodeS-15B|Spider 2.0-Lite", "source_date": "Nov 2, 2024" } ] }, { "id": "screenspot-pro", "name": "ScreenSpot-Pro", "group": "Agent 与工具", "unit": "%", "metric": "Grounding accuracy", "protocol": { "en": "Professional GUI grounding; published micro-average with greedy decoding. Zoom, routing and agent configurations are retained.", "zh": "专业 GUI 定位；使用官方贪心解码的微平均成绩，保留缩放、路由和 Agent 配置。" }, "source": { "data_url": "https://gui-agent.github.io/grounding-leaderboard/results/screenspot_pro.json", "sha256": "78bdb52432e27d4814ad944e5c11d7322b06b0d26729c446ef7c841460cf2788", "retrieved_at": "2026-11-01T16:40:34.874903+00:00", "name": "ScreenSpot-Pro", "url": "https://gui-agent.github.io/grounding-leaderboard/", "selector": "results.overall.avg (micro-average); not ScreenSpot v2", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "4e588f29430efc28b150", "model": "GroundCascade (Indeed-UI-32B + KV-Ground-8B)", "organization": "", "configuration": "Training-free geometric consistency cascade; Indeed-UI-32B primary/verifier + conditional KV-Ground-8B helper; fixed 0.5 zoom; 1% self-consistency gate; 4% cross-model agreement gate; greedy decoding; 2.574 average model calls/example.", "score": 0.8513598987982289, "raw_score": 0.8513598987982289, "raw_scale": 1, "source_record": "GroundCascade (Indeed-UI-32B + KV-Ground-8B)|Training-free geometric consistency cascade; Indeed-UI-32B primary/verifier + conditional KV-Ground-8B helper; fixed 0.5 zoom; 1% self-consistency gate; 4% cross-model agreement gate; greedy decoding; 2.574 average model calls/example." }, { "id": "b13606466954042b5fe2", "model": "GroundCascade (Indeed-UI-32B + Indeed-UI-8B)", "organization": "", "configuration": "Training-free geometric consistency cascade; Indeed-UI-32B primary/verifier + conditional Indeed-UI-8B helper; fixed 0.5 zoom; 1% self-consistency gate; 4% cross-model agreement gate; greedy decoding; 2.572 average model calls/example.", "score": 0.8500948766603416, "raw_score": 0.8500948766603416, "raw_scale": 1, "source_record": "GroundCascade (Indeed-UI-32B + Indeed-UI-8B)|Training-free geometric consistency cascade; Indeed-UI-32B primary/verifier + conditional Indeed-UI-8B helper; fixed 0.5 zoom; 1% self-consistency gate; 4% cross-model agreement gate; greedy decoding; 2.572 average model calls/example." }, { "id": "5a46fbd8d5e81d1ad64e", "model": "Indeed-UI-32B-zoomin", "organization": "", "configuration": "", "score": 0.8273244781783681, "raw_score": 0.8273244781783681, "raw_scale": 1, "source_record": "Indeed-UI-32B-zoomin|" }, { "id": "edf19e61e1aaca240532", "model": "Indeed-UI-8B-zoom-in", "organization": "", "configuration": "", "score": 0.8096141682479443, "raw_score": 0.8096141682479443, "raw_scale": 1, "source_record": "Indeed-UI-8B-zoom-in|" }, { "id": "104b58e62ed5e5153522", "model": "KV-Ground-8B + Qwen3.5-27B Consistency Router", "organization": "", "configuration": "Heterogeneous ensemble of KV-Ground-8B-BaseGuiOwl1.5-0315 and Qwen3.5-27B-AWQ-4bit. Both models run independent 2-step zoom-in pipelines (crop_ratio=0.5). Per sample, the model with lower zoom consistency (step-2 prediction closer to crop center = higher confidence) is selected. KV-Ground selected for 64.9% of samples, Qwen3.5-27B for 35.1%.", "score": 0.809, "raw_score": 0.809, "raw_scale": 1, "source_record": "KV-Ground-8B + Qwen3.5-27B Consistency Router|Heterogeneous ensemble of KV-Ground-8B-BaseGuiOwl1.5-0315 and Qwen3.5-27B-AWQ-4bit. Both models run independent 2-step zoom-in pipelines (crop_ratio=0.5). Per sample, the model with lower zoom consistency (step-2 prediction closer to crop center = higher confidence) is selected. KV-Ground selected for 64.9% of samples, Qwen3.5-27B for 35.1%." }, { "id": "998d02abebb38342204a", "model": "KV-Ground-GuiOwl1.5-0315-8B-ZoomIn", "organization": "", "configuration": "", "score": 0.8051865907653384, "raw_score": 0.8051865907653384, "raw_scale": 1, "source_record": "KV-Ground-GuiOwl1.5-0315-8B-ZoomIn|" }, { "id": "d274b159b1d4d47534c5", "model": "Holo2-235B-A22B (Agentic)", "organization": "", "configuration": "Holo2-235B-A22B (Agentic)", "score": 0.7849462365591398, "raw_score": 0.7849462365591398, "raw_scale": 1, "source_record": "Holo2-235B-A22B (Agentic)|Holo2-235B-A22B (Agentic)" }, { "id": "ce6108e9bf1276493038", "model": "MAI-UI-32B (MVP)", "organization": "", "configuration": "MVP on attention layer 48; max inference process is 2.", "score": 0.7746212121212122, "raw_score": 0.7746212121212122, "raw_scale": 1, "source_record": "MAI-UI-32B (MVP)|MVP on attention layer 48; max inference process is 2." }, { "id": "e84a981ac37ea687b105", "model": "AdaZoom-GUI-4B (cond. zoom-in, refine w/ Qwen3.5-397B)", "organization": "", "configuration": "Conditional zoom-in and refine with Qwen3.5-397B-A17B", "score": 0.7678684376976597, "raw_score": 0.7678684376976597, "raw_scale": 1, "source_record": "AdaZoom-GUI-4B (cond. zoom-in, refine w/ Qwen3.5-397B)|Conditional zoom-in and refine with Qwen3.5-397B-A17B" }, { "id": "e74e0e85d382f6a99811", "model": "KV-Ground-GuiOwl1.5-4B-0228-ZoomIn", "organization": "", "configuration": "", "score": 0.7640733712839974, "raw_score": 0.7640733712839974, "raw_scale": 1, "source_record": "KV-Ground-GuiOwl1.5-4B-0228-ZoomIn|" }, { "id": "805a964219b19202a637", "model": "Holo2-30B-A3B (Agentic)", "organization": "", "configuration": "Holo2-30B-A3B (Agentic)", "score": 0.7520556609740671, "raw_score": 0.7520556609740671, "raw_scale": 1, "source_record": "Holo2-30B-A3B (Agentic)|Holo2-30B-A3B (Agentic)" }, { "id": "4b869a0c3ff8a85b8e44", "model": "MVP_Qwen3VL-32B", "organization": "", "configuration": "MVP crops different regions from original screenshot to get different prediction and then cluster them through k-means. The final result is decided by the centroid coordinate of the largest cluster.", "score": 0.7414664981036663, "raw_score": 0.7414664981036663, "raw_scale": 1, "source_record": "MVP_Qwen3VL-32B|MVP crops different regions from original screenshot to get different prediction and then cluster them through k-means. The final result is decided by the centroid coordinate of the largest cluster." }, { "id": "b42ca1e9c5994d16d5d2", "model": "MAI-UI-32B (Zoom In)", "organization": "", "configuration": "", "score": 0.734977862112587, "raw_score": 0.734977862112587, "raw_scale": 1, "source_record": "MAI-UI-32B (Zoom In)|" }, { "id": "e477853e06ad76740bcb", "model": "Indeed-UI-8B", "organization": "", "configuration": "", "score": 0.7337128399746996, "raw_score": 0.7337128399746996, "raw_scale": 1, "source_record": "Indeed-UI-8B|" }, { "id": "c379b8b3bdfc05a75aea", "model": "Indeed-UI-32B", "organization": "", "configuration": "", "score": 0.7330803289057558, "raw_score": 0.7330803289057558, "raw_scale": 1, "source_record": "Indeed-UI-32B|" }, { "id": "668d1a2f5ee2b764b071", "model": "KV-Ground-GuiOwl1.5-0315-8B", "organization": "", "configuration": "", "score": 0.7318153067678684, "raw_score": 0.7318153067678684, "raw_scale": 1, "source_record": "KV-Ground-GuiOwl1.5-0315-8B|" }, { "id": "d2a6697a87cbb000045f", "model": "Duvo Eye-1 (Holo-3.1-35B-A3B + LoRA)", "organization": "", "configuration": "Holo-3.1-35B-A3B with a LoRA adapter (rank 64, alpha 128, 1 epoch) trained on duvoai/SynthUI (14.9k rows). Single-shot grounding, temperature 0, thinking disabled, guided-JSON {\"x\",\"y\"} in [0,1000] scaled to absolute pixels. Self-reported; per-sample predictions published for rescoring at https://huggingface.co/datasets/duvoai/duvo-eye-1-evals", "score": 0.7293, "raw_score": 0.7293, "raw_scale": 1, "source_record": "Duvo Eye-1 (Holo-3.1-35B-A3B + LoRA)|Holo-3.1-35B-A3B with a LoRA adapter (rank 64, alpha 128, 1 epoch) trained on duvoai/SynthUI (14.9k rows). Single-shot grounding, temperature 0, thinking disabled, guided-JSON {\"x\",\"y\"} in [0,1000] scaled to absolute pixels. Self-reported; per-sample predictions published for rescoring at https://huggingface.co/datasets/duvoai/duvo-eye-1-evals" }, { "id": "5401c407c9eefe7f4b2b", "model": "HuzzleWorld-3B (Zoom In)", "organization": "", "configuration": "Proprietary", "score": 0.7242, "raw_score": 0.7242, "raw_scale": 1, "source_record": "HuzzleWorld-3B (Zoom In)|Proprietary" }, { "id": "fc88aac18a4d2ebe742c", "model": "MAI-UI-8B (Zoom In)", "organization": "", "configuration": "", "score": 0.7191650853889943, "raw_score": 0.7191650853889943, "raw_scale": 1, "source_record": "MAI-UI-8B (Zoom In)|" }, { "id": "0b64b43dae1cc133348f", "model": "Holo2-8B (Agentic)", "organization": "", "configuration": "Holo2-8B (Agentic)", "score": 0.7141049968374447, "raw_score": 0.7141049968374447, "raw_scale": 1, "source_record": "Holo2-8B (Agentic)|Holo2-8B (Agentic)" }, { "id": "41c0fe3691dab66e4a88", "model": "HuzzleWorld-2B (Zoom In)", "organization": "", "configuration": "Proprietary", "score": 0.7084, "raw_score": 0.7084, "raw_scale": 1, "source_record": "HuzzleWorld-2B (Zoom In)|Proprietary" }, { "id": "b52a83e7911a808999c1", "model": "AdaZoom-GUI-4B (conditional zoom-in)", "organization": "", "configuration": "", "score": 0.7058823529411765, "raw_score": 0.7058823529411765, "raw_scale": 1, "source_record": "AdaZoom-GUI-4B (conditional zoom-in)|" }, { "id": "7c162688a280ccaea478", "model": "Holo2-235B-A22B", "organization": "", "configuration": "Holo2-235B-A22B", "score": 0.7058823529411765, "raw_score": 0.7058823529411765, "raw_scale": 1, "source_record": "Holo2-235B-A22B|Holo2-235B-A22B" }, { "id": "fa05b4488c2590640197", "model": "KV-Ground-Qwen3VL-4B-ZoomIn", "organization": "", "configuration": "", "score": 0.702719797596458, "raw_score": 0.702719797596458, "raw_scale": 1, "source_record": "KV-Ground-Qwen3VL-4B-ZoomIn|" }, { "id": "28d70a2333513cc03f87", "model": "UI-Venus-1-5-30B-A3B", "organization": "", "configuration": "", "score": 0.6963946869070209, "raw_score": 0.6963946869070209, "raw_scale": 1, "source_record": "UI-Venus-1-5-30B-A3B|" }, { "id": "ac1575b74f3dc4f0db62", "model": "Holo2-4B (Agentic)", "organization": "", "configuration": "Holo2-4B (Agentic)", "score": 0.6856419987349779, "raw_score": 0.6856419987349779, "raw_scale": 1, "source_record": "Holo2-4B (Agentic)|Holo2-4B (Agentic)" }, { "id": "08ec647961ef8f51fe3a", "model": "UI-Venus-1-5-8B", "organization": "", "configuration": "", "score": 0.6843769765970904, "raw_score": 0.6843769765970904, "raw_scale": 1, "source_record": "UI-Venus-1-5-8B|" }, { "id": "cd94338dcd0b6022daf7", "model": "MAI-UI-32B", "organization": "", "configuration": "", "score": 0.6786843769765971, "raw_score": 0.6786843769765971, "raw_scale": 1, "source_record": "MAI-UI-32B|" }, { "id": "976582350ebc052a3bb1", "model": "KV-Ground-GuiOwl1.5-0228-4B", "organization": "", "configuration": "", "score": 0.6704617330803289, "raw_score": 0.6704617330803289, "raw_scale": 1, "source_record": "KV-Ground-GuiOwl1.5-0228-4B|" }, { "id": "073aaef9778e5a4a8b7a", "model": "HuzzleWorld-3B", "organization": "", "configuration": "Proprietary", "score": 0.6641, "raw_score": 0.6641, "raw_scale": 1, "source_record": "HuzzleWorld-3B|Proprietary" }, { "id": "98d39e8b4894314626db", "model": "Holo2-30B-A3B", "organization": "", "configuration": "", "score": 0.6609740670461733, "raw_score": 0.6609740670461733, "raw_scale": 1, "source_record": "Holo2-30B-A3B|" }, { "id": "d68496c388a4e7a0963a", "model": "MAI-UI-8B", "organization": "", "configuration": "", "score": 0.6565464895635673, "raw_score": 0.6565464895635673, "raw_scale": 1, "source_record": "MAI-UI-8B|" }, { "id": "982cedd6b620c1a87902", "model": "MVP_Qwen3VL-8B", "organization": "", "configuration": "MVP crops different regions from original screenshot to get different prediction and then cluster them through k-means. The final result is decided by the centroid coordinate of the largest cluster.", "score": 0.6502525252525253, "raw_score": 0.6502525252525253, "raw_scale": 1, "source_record": "MVP_Qwen3VL-8B|MVP crops different regions from original screenshot to get different prediction and then cluster them through k-means. The final result is decided by the centroid coordinate of the largest cluster." }, { "id": "89171037232d1d61c542", "model": "GTA1-32B", "organization": "", "configuration": "", "score": 0.6363061353573688, "raw_score": 0.6363061353573688, "raw_scale": 1, "source_record": "GTA1-32B|" }, { "id": "07c97a89578d1c6a029f", "model": "HuzzleWorld-1B (Zoom In)", "organization": "", "configuration": "Proprietary", "score": 0.635, "raw_score": 0.635, "raw_scale": 1, "source_record": "HuzzleWorld-1B (Zoom In)|Proprietary" }, { "id": "1098afaaa778d219580e", "model": "Holo1.5-72B", "organization": "", "configuration": "", "score": 0.6325110689437066, "raw_score": 0.6325110689437066, "raw_scale": 1, "source_record": "Holo1.5-72B|" }, { "id": "69fbe481d447a1ba0164", "model": "KV-Ground-Qwen3VL-4B", "organization": "", "configuration": "", "score": 0.6318785578747628, "raw_score": 0.6318785578747628, "raw_scale": 1, "source_record": "KV-Ground-Qwen3VL-4B|" }, { "id": "cacdeb83806056332e20", "model": "MAI-UI-2B (Zoom In)", "organization": "", "configuration": "", "score": 0.6280834914611005, "raw_score": 0.6280834914611005, "raw_scale": 1, "source_record": "MAI-UI-2B (Zoom In)|" }, { "id": "c17448e286864b8cad35", "model": "UI-Venus-72B", "organization": "", "configuration": "", "score": 0.618595825426945, "raw_score": 0.618595825426945, "raw_scale": 1, "source_record": "UI-Venus-72B|" }, { "id": "41e1f858c1ed5ca632d3", "model": "HuzzleWorld-2B", "organization": "", "configuration": "Proprietary", "score": 0.6173, "raw_score": 0.6173, "raw_scale": 1, "source_record": "HuzzleWorld-2B|Proprietary" }, { "id": "0bd5068650decf9e80cc", "model": "AdaZoom-GUI-4B", "organization": "", "configuration": "", "score": 0.6160657811511702, "raw_score": 0.6160657811511702, "raw_scale": 1, "source_record": "AdaZoom-GUI-4B|" }, { "id": "bcf4c6e6762848b267e3", "model": "UI-TARS-1.5", "organization": "", "configuration": "", "score": 0.6160657811511702, "raw_score": 0.6160657811511702, "raw_scale": 1, "source_record": "UI-TARS-1.5|" }, { "id": "a97ddd113c6213171525", "model": "Seed-1.5-VL", "organization": "", "configuration": "", "score": 0.6091081593927894, "raw_score": 0.6091081593927894, "raw_scale": 1, "source_record": "Seed-1.5-VL|" }, { "id": "732fd711e01e40534fcd", "model": "GUI-ARP-7B", "organization": "", "configuration": "", "score": 0.6078, "raw_score": 0.6078, "raw_scale": 1, "source_record": "GUI-ARP-7B|" }, { "id": "1fc95f3943269537497c", "model": "GUI-AIMA-3B", "organization": "", "configuration": "", "score": 0.5958254269449715, "raw_score": 0.5958254269449715, "raw_scale": 1, "source_record": "GUI-AIMA-3B|" }, { "id": "1e3bd1521156184f3c81", "model": "Holo2-8B", "organization": "", "configuration": "", "score": 0.5888678051865908, "raw_score": 0.5888678051865908, "raw_scale": 1, "source_record": "Holo2-8B|" }, { "id": "abe20acb0572e459c7a7", "model": "GTA1-Qwen2.5VL-72B", "organization": "", "configuration": "", "score": 0.5844402277039848, "raw_score": 0.5844402277039848, "raw_scale": 1, "source_record": "GTA1-Qwen2.5VL-72B|" }, { "id": "fdd3f359fe5c83269713", "model": "Holo1.5-7B", "organization": "", "configuration": "", "score": 0.5793801391524351, "raw_score": 0.5793801391524351, "raw_scale": 1, "source_record": "Holo1.5-7B|" }, { "id": "a4bb20a2405e094afefe", "model": "BAMI-7B", "organization": "", "configuration": "A test-time inference method for GUI grounding that uses coarse-to-fine focus and candidate selection. Base model: TianXi-Action-7B.", "score": 0.5774826059456041, "raw_score": 0.5774826059456041, "raw_scale": 1, "source_record": "BAMI-7B|A test-time inference method for GUI grounding that uses coarse-to-fine focus and candidate selection. Base model: TianXi-Action-7B." }, { "id": "6098bb473985b7b4b557", "model": "UI-Venus-1-5-2B", "organization": "", "configuration": "", "score": 0.5774826059456041, "raw_score": 0.5774826059456041, "raw_scale": 1, "source_record": "UI-Venus-1-5-2B|" }, { "id": "013c9617f5a848b29885", "model": "MAI-UI-2B", "organization": "", "configuration": "", "score": 0.5736875395319418, "raw_score": 0.5736875395319418, "raw_scale": 1, "source_record": "MAI-UI-2B|" }, { "id": "b8c95e53b665ba57f669", "model": "Holo2-4B", "organization": "", "configuration": "", "score": 0.5724225173940544, "raw_score": 0.5724225173940544, "raw_scale": 1, "source_record": "Holo2-4B|" }, { "id": "17988dcdd836f591f95a", "model": "HuzzleWorld-1B", "organization": "", "configuration": "Proprietary", "score": 0.556, "raw_score": 0.556, "raw_scale": 1, "source_record": "HuzzleWorld-1B|Proprietary" }, { "id": "c6b0d7a5fe016a7710ee", "model": "GTA1-7B", "organization": "", "configuration": "", "score": 0.5547122074636306, "raw_score": 0.5547122074636306, "raw_scale": 1, "source_record": "GTA1-7B|" }, { "id": "dce8346a098888458ee7", "model": "GTA1-Qwen2.5VL-32B", "organization": "", "configuration": "", "score": 0.5357368753953194, "raw_score": 0.5357368753953194, "raw_scale": 1, "source_record": "GTA1-Qwen2.5VL-32B|" }, { "id": "0f9861d083f259627578", "model": "Qwen2.5-VL-72B-Instruct", "organization": "Alibaba", "configuration": "", "score": 0.5332068311195446, "raw_score": 0.5332068311195446, "raw_scale": 1, "source_record": "Qwen2.5-VL-72B-Instruct|" }, { "id": "2cb463c4b490da938757", "model": "GUI-Spotlight", "organization": "", "configuration": "A think-with-image GUI visual grounding model.", "score": 0.528147, "raw_score": 0.528147, "raw_scale": 1, "source_record": "GUI-Spotlight|A think-with-image GUI visual grounding model." }, { "id": "f19088c58b1ca4056d1e", "model": "V2P-7B", "organization": "", "configuration": "", "score": 0.5253164556962026, "raw_score": 0.5253164556962026, "raw_scale": 1, "source_record": "V2P-7B|" }, { "id": "2aced166c828424345a7", "model": "TianXi-Action-7B", "organization": "", "configuration": "", "score": 0.5186590765338394, "raw_score": 0.5186590765338394, "raw_scale": 1, "source_record": "TianXi-Action-7B|" }, { "id": "2d6ede6cae159d5dd71a", "model": "Holo1.5-3B", "organization": "", "configuration": "", "score": 0.5148640101201771, "raw_score": 0.5148640101201771, "raw_scale": 1, "source_record": "Holo1.5-3B|" }, { "id": "032ba29b5c99015626ad", "model": "Kimi-VL-A3B-Thinking-2506", "organization": "Moonshot AI", "configuration": "", "score": 0.5098039215686274, "raw_score": 0.5098039215686274, "raw_scale": 1, "source_record": "Kimi-VL-A3B-Thinking-2506|" }, { "id": "7bd0fe9f2160ac1f6a22", "model": "UI-Venus-7B", "organization": "", "configuration": "", "score": 0.5079063883617964, "raw_score": 0.5079063883617964, "raw_scale": 1, "source_record": "UI-Venus-7B|" }, { "id": "6e3ff4513be4ca28ed89", "model": "GTA1-Qwen2.5VL-7B", "organization": "", "configuration": "", "score": 0.5009487666034156, "raw_score": 0.5009487666034156, "raw_scale": 1, "source_record": "GTA1-Qwen2.5VL-7B|" }, { "id": "66ef2d29c4a63203c820", "model": "Qwen2.5-VL-32B-Instruct", "organization": "Alibaba", "configuration": "", "score": 0.48007590132827327, "raw_score": 0.48007590132827327, "raw_scale": 1, "source_record": "Qwen2.5-VL-32B-Instruct|" }, { "id": "b8bc7726a7faeb14b976", "model": "UI-AGILE-7B", "organization": "", "configuration": "UI-AGILE enhances GUI agents through improved training with a Continuous Reward function, and Cropping-based Resampling, and inference with Decomposed Grounding with Selection.", "score": 0.47881087919038584, "raw_score": 0.47881087919038584, "raw_scale": 1, "source_record": "UI-AGILE-7B|UI-AGILE enhances GUI agents through improved training with a Continuous Reward function, and Cropping-based Resampling, and inference with Decomposed Grounding with Selection." }, { "id": "900e54590ad06bd69c38", "model": "SE-GUI-7B", "organization": "", "configuration": "", "score": 0.47248576850094876, "raw_score": 0.47248576850094876, "raw_scale": 1, "source_record": "SE-GUI-7B|" }, { "id": "4cb2885b7b9ce76e0e4c", "model": "UI-AGILE-3B", "organization": "", "configuration": "UI-AGILE enhances GUI agents through improved training with a Continuous Reward function, and Cropping-based Resampling, and inference with Decomposed Grounding with Selection.", "score": 0.4497153700189753, "raw_score": 0.4497153700189753, "raw_scale": 1, "source_record": "UI-AGILE-3B|UI-AGILE enhances GUI agents through improved training with a Continuous Reward function, and Cropping-based Resampling, and inference with Decomposed Grounding with Selection." }, { "id": "f036a1e2f196fb92fb27", "model": "GUI-Actor-2.5VL-7B", "organization": "", "configuration": "", "score": 0.4459203036053131, "raw_score": 0.4459203036053131, "raw_scale": 1, "source_record": "GUI-Actor-2.5VL-7B|" }, { "id": "4fc2f72181d53bd00861", "model": "GUI-Actor-2.5VL-3B", "organization": "", "configuration": "", "score": 0.42188488298545224, "raw_score": 0.42188488298545224, "raw_scale": 1, "source_record": "GUI-Actor-2.5VL-3B|" }, { "id": "1e21a634c8790a334f31", "model": "GUI-Actor-2VL-7B", "organization": "", "configuration": "", "score": 0.407337128399747, "raw_score": 0.407337128399747, "raw_scale": 1, "source_record": "GUI-Actor-2VL-7B|" }, { "id": "2e6812eff5d6a934e8b4", "model": "UI-TARS-72B", "organization": "", "configuration": "", "score": 0.38140417457305503, "raw_score": 0.38140417457305503, "raw_scale": 1, "source_record": "UI-TARS-72B|" }, { "id": "c97059b569c0de589bbe", "model": "GUI-Actor-2VL-2B", "organization": "", "configuration": "", "score": 0.3668564199873498, "raw_score": 0.3668564199873498, "raw_scale": 1, "source_record": "GUI-Actor-2VL-2B|" }, { "id": "17645a7e5ab87d67285d", "model": "GUI-G1", "organization": "", "configuration": "", "score": 0.3592662871600253, "raw_score": 0.3592662871600253, "raw_scale": 1, "source_record": "GUI-G1|" }, { "id": "0735628dd5d6b20ee23e", "model": "SE-GUI-3B", "organization": "", "configuration": "", "score": 0.35863377609108, "raw_score": 0.35863377609108, "raw_scale": 1, "source_record": "SE-GUI-3B|" }, { "id": "009d94df3a38497c6315", "model": "UI-TARS-7B", "organization": "", "configuration": "", "score": 0.3573687539531942, "raw_score": 0.3573687539531942, "raw_scale": 1, "source_record": "UI-TARS-7B|" }, { "id": "c8d6c227d43a0da395b2", "model": "UGround-v1-72B", "organization": "", "configuration": "", "score": 0.34535104364326374, "raw_score": 0.34535104364326374, "raw_scale": 1, "source_record": "UGround-v1-72B|" }, { "id": "59eed0f807a3a0ca4e64", "model": "Di-GUI-3B", "organization": "", "configuration": "", "score": 0.31562302340290954, "raw_score": 0.31562302340290954, "raw_scale": 1, "source_record": "Di-GUI-3B|" }, { "id": "4a553b81679b097a06cb", "model": "UGround-V1-7B", "organization": "", "configuration": "", "score": 0.3105629348513599, "raw_score": 0.3105629348513599, "raw_scale": 1, "source_record": "UGround-V1-7B|" }, { "id": "b8c3ceabb1d6cb9fb57a", "model": "ZonUI-3B", "organization": "", "configuration": "", "score": 0.28716002530044277, "raw_score": 0.28716002530044277, "raw_scale": 1, "source_record": "ZonUI-3B|" }, { "id": "babd0d552050a2e50a67", "model": "UI-TARS-2B", "organization": "", "configuration": "", "score": 0.27703984819734345, "raw_score": 0.27703984819734345, "raw_scale": 1, "source_record": "UI-TARS-2B|" }, { "id": "0f1d757bf674c11cf2ea", "model": "Qwen2.5-VL-7B-Instruct", "organization": "Alibaba", "configuration": "", "score": 0.2675521821631879, "raw_score": 0.2675521821631879, "raw_scale": 1, "source_record": "Qwen2.5-VL-7B-Instruct|" }, { "id": "23735bd7d18692157a47", "model": "UGround-V1-2B", "organization": "", "configuration": "", "score": 0.26628716002530045, "raw_score": 0.26628716002530045, "raw_scale": 1, "source_record": "UGround-V1-2B|" }, { "id": "0736d140acef688526cf", "model": "Aguvis-7B", "organization": "", "configuration": "", "score": 0.22896900695762176, "raw_score": 0.22896900695762176, "raw_scale": 1, "source_record": "Aguvis-7B|" }, { "id": "ebc6198582b5798df221", "model": "OS-Atlas-7B", "organization": "", "configuration": "", "score": 0.18912080961416824, "raw_score": 0.18912080961416824, "raw_scale": 1, "source_record": "OS-Atlas-7B|" }, { "id": "68efb0129ca5cc4e12da", "model": "GPT5-minimal (resized)", "organization": "OpenAI", "configuration": "", "score": 0.18532574320050602, "raw_score": 0.18532574320050602, "raw_scale": 1, "source_record": "GPT5-minimal (resized)|" }, { "id": "8f99397daaacb7d9a85f", "model": "Claude (Computer Use)", "organization": "Anthropic", "configuration": "", "score": 0.17101226993865032, "raw_score": 0.17101226993865032, "raw_scale": 1, "source_record": "Claude (Computer Use)|" }, { "id": "e1e17c97be3193bcd288", "model": "UGround-7B", "organization": "", "configuration": "", "score": 0.1650853889943074, "raw_score": 0.1650853889943074, "raw_scale": 1, "source_record": "UGround-7B|" }, { "id": "707ee4ceb88832c8825a", "model": "Qwen2.5-VL-3B-Instruct", "organization": "Alibaba", "configuration": "", "score": 0.16065781151170144, "raw_score": 0.16065781151170144, "raw_scale": 1, "source_record": "Qwen2.5-VL-3B-Instruct|" }, { "id": "e7454b4a88cd3e22c3d3", "model": "AriaUI (MOE, 3.9B active)", "organization": "", "configuration": "", "score": 0.11321948134092347, "raw_score": 0.11321948134092347, "raw_scale": 1, "source_record": "AriaUI (MOE, 3.9B active)|" }, { "id": "52a4c68178d3bb0d42b2", "model": "CogAgent (18B)", "organization": "", "configuration": "", "score": 0.07716635041113219, "raw_score": 0.07716635041113219, "raw_scale": 1, "source_record": "CogAgent (18B)|" }, { "id": "6d323f908958b36c7d75", "model": "ShowUI (2B)", "organization": "", "configuration": "", "score": 0.07716635041113219, "raw_score": 0.07716635041113219, "raw_scale": 1, "source_record": "ShowUI (2B)|" }, { "id": "6f9bcd04efb01c089f69", "model": "GPT5-high (resized)", "organization": "OpenAI", "configuration": "", "score": 0.06008855154965212, "raw_score": 0.06008855154965212, "raw_scale": 1, "source_record": "GPT5-high (resized)|" }, { "id": "04710b8becb15936148e", "model": "OS-Atlas-4B", "organization": "", "configuration": "", "score": 0.03731815306767868, "raw_score": 0.03731815306767868, "raw_scale": 1, "source_record": "OS-Atlas-4B|" }, { "id": "1c7269bcc9d497b2588d", "model": "MiniCPM-V (7B)", "organization": "", "configuration": "", "score": 0.030360531309297913, "raw_score": 0.030360531309297913, "raw_scale": 1, "source_record": "MiniCPM-V (7B)|" }, { "id": "a77828036c03b560bfdd", "model": "Qwen2-VL-7B", "organization": "Alibaba", "configuration": "", "score": 0.015812776723592662, "raw_score": 0.015812776723592662, "raw_scale": 1, "source_record": "Qwen2-VL-7B|" }, { "id": "62df2d80fe1e0fff5d91", "model": "SeeClick (7B)", "organization": "", "configuration": "", "score": 0.011385199240986717, "raw_score": 0.011385199240986717, "raw_scale": 1, "source_record": "SeeClick (7B)|" }, { "id": "307961b49bdb3bfc4152", "model": "Qwen2-VL-72B-Instruct", "organization": "Alibaba", "configuration": "", "score": 0.010120177103099304, "raw_score": 0.010120177103099304, "raw_scale": 1, "source_record": "Qwen2-VL-72B-Instruct|" }, { "id": "2e797fd6395bd849020a", "model": "GPT-4o", "organization": "OpenAI", "configuration": "", "score": 0.008222643896268185, "raw_score": 0.008222643896268185, "raw_scale": 1, "source_record": "GPT-4o|" }, { "id": "27c8e73665c2b949ff22", "model": "Qwen-VL-7B", "organization": "Alibaba", "configuration": "", "score": 0.0006325110689437065, "raw_score": 0.0006325110689437065, "raw_scale": 1, "source_record": "Qwen-VL-7B|" } ] }, { "id": "mathvista-testmini", "name": "MathVista (testmini)", "group": "数学与科学", "unit": "%", "metric": "Accuracy", "protocol": { "en": "1,000-question testmini cohort, not the full test split. Prompting and tool methods follow the official results.", "zh": "1,000 道题的 testmini，不与完整测试集混用；保留官方成绩中的提示与工具方法。" }, "source": { "data_url": "https://mathvista.github.io/data/results/model_scores.js", "sha256": "5bf339037c2989a809735fbb8324f63a5d4776e806e2dc084fc1e9bcbd302e64", "retrieved_at": "2026-11-01T16:39:18.175255+00:00", "name": "MathVista", "url": "https://mathvista.github.io/", "selector": "score_table: ALL; testmini; exclude human/random", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "abb151638600758fa0a0", "model": "DreamPRM (o4-mini)", "organization": "", "configuration": "Reason 🧠", "score": 0.852, "raw_score": 85.2, "raw_scale": 100, "source_record": "DreamPRM (o4-mini)|Reason 🧠", "source_date": "2025-06-04" }, { "id": "2b5dd63efe5d9c12cb98", "model": "VL-Rethinker", "organization": "", "configuration": "Reason 🧠", "score": 0.8029999999999999, "raw_score": 80.3, "raw_scale": 100, "source_record": "VL-Rethinker|Reason 🧠", "source_date": "2025-04-10" }, { "id": "f3f537e400c5698d15d2", "model": "Step R1-V-Mini", "organization": "", "configuration": "Reason 🧠", "score": 0.8009999999999999, "raw_score": 80.1, "raw_scale": 100, "source_record": "Step R1-V-Mini|Reason 🧠", "source_date": "2025-04-07" }, { "id": "16857d0627fa2fe91cc5", "model": "Kimi-k1.6-preview-20250308", "organization": "Moonshot AI", "configuration": "Reason 🧠", "score": 0.8, "raw_score": 80.0, "raw_scale": 100, "source_record": "Kimi-k1.6-preview-20250308|Reason 🧠", "source_date": "2025-03-10" }, { "id": "36fec394822845f7e625", "model": "Doubao-pro-1.5", "organization": "", "configuration": "Reason 🧠", "score": 0.795, "raw_score": 79.5, "raw_scale": 100, "source_record": "Doubao-pro-1.5|Reason 🧠", "source_date": "2025-01-22" }, { "id": "d9b60ed52618a910e400", "model": "Ovis2_34B", "organization": "", "configuration": "LMM 🖼️", "score": 0.7709999999999999, "raw_score": 77.1, "raw_scale": 100, "source_record": "Ovis2_34B|LMM 🖼️", "source_date": "2025-02-10" }, { "id": "33f82ae7c0958e416ed7", "model": "Kimi-k1.5", "organization": "Moonshot AI", "configuration": "Reason 🧠", "score": 0.7490000000000001, "raw_score": 74.9, "raw_scale": 100, "source_record": "Kimi-k1.5|Reason 🧠", "source_date": "2025-01-22" }, { "id": "f4d1004862c356a77134", "model": "OpenAI o1", "organization": "OpenAI", "configuration": "Reason 🧠", "score": 0.7390000000000001, "raw_score": 73.9, "raw_scale": 100, "source_record": "OpenAI o1|Reason 🧠", "source_date": "2024-09-12" }, { "id": "0f9b3da8d3bdcad6d7fc", "model": "Llama 4 Maverick", "organization": "Meta", "configuration": "LMM 🖼️", "score": 0.737, "raw_score": 73.7, "raw_scale": 100, "source_record": "Llama 4 Maverick|LMM 🖼️", "source_date": "2025-04-05" }, { "id": "eae81a0d02ffad9cdd2c", "model": "Vision-R1-7B", "organization": "", "configuration": "Reason 🧠", "score": 0.732, "raw_score": 73.2, "raw_scale": 100, "source_record": "Vision-R1-7B|Reason 🧠", "source_date": "2025-03-09" }, { "id": "c404208f5a2fe36e505c", "model": "Gemini 2.0 Flash", "organization": "Google", "configuration": "LMM 🖼️", "score": 0.731, "raw_score": 73.1, "raw_scale": 100, "source_record": "Gemini 2.0 Flash|LMM 🖼️", "source_date": "2025-02-05" }, { "id": "890e535674abb4f6d0da", "model": "QVQ-72B-Preview", "organization": "", "configuration": "LMM 🖼️", "score": 0.7140000000000001, "raw_score": 71.4, "raw_scale": 100, "source_record": "QVQ-72B-Preview|LMM 🖼️", "source_date": "2024-12-24" }, { "id": "0657215fff5ef9e8c675", "model": "Qwen2VL-72B", "organization": "Alibaba", "configuration": "LMM 🖼️", "score": 0.705, "raw_score": 70.5, "raw_scale": 100, "source_record": "Qwen2VL-72B|LMM 🖼️", "source_date": "2024-12-24" }, { "id": "7d4fd242527a5e549e92", "model": "Pixtral Large (124B)", "organization": "", "configuration": "LMM 🖼️", "score": 0.6940000000000001, "raw_score": 69.4, "raw_scale": 100, "source_record": "Pixtral Large (124B)|LMM 🖼️", "source_date": "2024-11-18" }, { "id": "1a037fece5ebae30e002", "model": "Grok-2", "organization": "xAI", "configuration": "LMM 🖼️", "score": 0.69, "raw_score": 69.0, "raw_scale": 100, "source_record": "Grok-2|LMM 🖼️", "source_date": "2024-08-13" }, { "id": "3e0628dcd16bd6f39569", "model": "Grok-2 mini", "organization": "xAI", "configuration": "LMM 🖼️", "score": 0.6809999999999999, "raw_score": 68.1, "raw_scale": 100, "source_record": "Grok-2 mini|LMM 🖼️", "source_date": "2024-08-13" }, { "id": "75f6b473c3ac694d63b7", "model": "Claude 3.5 Sonnet", "organization": "Anthropic", "configuration": "LMM 🖼️", "score": 0.677, "raw_score": 67.7, "raw_scale": 100, "source_record": "Claude 3.5 Sonnet|LMM 🖼️", "source_date": "2024-06-20" }, { "id": "d47a623d86ad829d433e", "model": "LLaVA-OneVision", "organization": "", "configuration": "LMM 🖼️", "score": 0.675, "raw_score": 67.5, "raw_scale": 100, "source_record": "LLaVA-OneVision|LMM 🖼️", "source_date": "2024-08-06" }, { "id": "f30b65d1aa46f0917d82", "model": "InternVL2-8B-MPO", "organization": "", "configuration": "LMM 🖼️", "score": 0.6729999999999999, "raw_score": 67.3, "raw_scale": 100, "source_record": "InternVL2-8B-MPO|LMM 🖼️", "source_date": "2024-11-14" }, { "id": "a77f2f04432f3672c752", "model": "Ovis1.6-Gemma2-9B", "organization": "", "configuration": "LMM 🖼️", "score": 0.672, "raw_score": 67.2, "raw_scale": 100, "source_record": "Ovis1.6-Gemma2-9B|LMM 🖼️", "source_date": "2025-02-10" }, { "id": "8dc39af0c58fef308565", "model": "InternVL2-Pro", "organization": "", "configuration": "LMM 🖼️", "score": 0.6679999999999999, "raw_score": 66.8, "raw_scale": 100, "source_record": "InternVL2-Pro|LMM 🖼️", "source_date": "2024-09-04" }, { "id": "05186d497931d12edfcd", "model": "Chimera-Reasoner-8B", "organization": "", "configuration": "LMM 🖼️", "score": 0.662, "raw_score": 66.2, "raw_scale": 100, "source_record": "Chimera-Reasoner-8B|LMM 🖼️", "source_date": "2024-12-05" }, { "id": "55f7941f52d96766200b", "model": "TextGrad (GPT-4o)", "organization": "", "configuration": "LMM 🖼️", "score": 0.6609999999999999, "raw_score": 66.1, "raw_scale": 100, "source_record": "TextGrad (GPT-4o)|LMM 🖼️", "source_date": "2024-07-08" }, { "id": "666273ee89ddf06809ea", "model": "TBAC-VLR1-3B-preview", "organization": "", "configuration": "LMM 🖼️", "score": 0.648, "raw_score": 64.8, "raw_scale": 100, "source_record": "TBAC-VLR1-3B-preview|LMM 🖼️", "source_date": "2025-04-16" }, { "id": "71f741fde0af4e1b8c3c", "model": "Gemini 1.5 Pro (May 2024)", "organization": "Google", "configuration": "LMM 🖼️", "score": 0.639, "raw_score": 63.9, "raw_scale": 100, "source_record": "Gemini 1.5 Pro (May 2024)|LMM 🖼️", "source_date": "2024-05-17" }, { "id": "a596b942aea3a468463f", "model": "GPT-4o", "organization": "OpenAI", "configuration": "LMM 🖼️", "score": 0.638, "raw_score": 63.8, "raw_scale": 100, "source_record": "GPT-4o|LMM 🖼️", "source_date": "2024-05-13" }, { "id": "f3a9b78d8b7c8b09c6fb", "model": "Phi-4-multimodal-ins", "organization": "Microsoft", "configuration": "LMM 🖼️", "score": 0.624, "raw_score": 62.4, "raw_scale": 100, "source_record": "Phi-4-multimodal-ins|LMM 🖼️", "source_date": "2024-03-03" }, { "id": "b97653807aeb9b7bcf80", "model": "InternVL-Chat-V1.2-Plus", "organization": "", "configuration": "LMM 🖼️", "score": 0.599, "raw_score": 59.9, "raw_scale": 100, "source_record": "InternVL-Chat-V1.2-Plus|LMM 🖼️", "source_date": "2024-02-22" }, { "id": "cc3ed47ccd1c730531f3", "model": "Gemini 1.5 Flash (May 2024)", "organization": "Google", "configuration": "LMM 🖼️", "score": 0.584, "raw_score": 58.4, "raw_scale": 100, "source_record": "Gemini 1.5 Flash (May 2024)|LMM 🖼️", "source_date": "2024-05-17" }, { "id": "7e70fcfbf18cfebee63c", "model": "GPT-4T 2024-04-09", "organization": "OpenAI", "configuration": "LMM 🖼️", "score": 0.581, "raw_score": 58.1, "raw_scale": 100, "source_record": "GPT-4T 2024-04-09|LMM 🖼️", "source_date": "2024-05-13" }, { "id": "bf92e87c0c1a6549e3d0", "model": "Pixtral-12B", "organization": "", "configuration": "LMM 🖼️", "score": 0.58, "raw_score": 58.0, "raw_scale": 100, "source_record": "Pixtral-12B|LMM 🖼️", "source_date": "2024-10-09" }, { "id": "1e3912af927dc1813e57", "model": "InternLM-XComposer2-VL-7B", "organization": "Shanghai AI Laboratory", "configuration": "LMM 🖼️", "score": 0.5760000000000001, "raw_score": 57.6, "raw_scale": 100, "source_record": "InternLM-XComposer2-VL-7B|LMM 🖼️", "source_date": "2024-01-22" }, { "id": "a7a4381a119045a8e011", "model": "Gemini 1.0 Ultra", "organization": "Google", "configuration": "LMM 🖼️", "score": 0.53, "raw_score": 53.0, "raw_scale": 100, "source_record": "Gemini 1.0 Ultra|LMM 🖼️", "source_date": "2023-12-06" }, { "id": "270077821a7168555c3a", "model": "Grok-1.5V", "organization": "xAI", "configuration": "LMM 🖼️", "score": 0.528, "raw_score": 52.8, "raw_scale": 100, "source_record": "Grok-1.5V|LMM 🖼️", "source_date": "2024-04-12" }, { "id": "d89e2b7205b4c5c4fc06", "model": "Gemini 1.5 Pro (Feb 2024)", "organization": "Google", "configuration": "LMM 🖼️", "score": 0.521, "raw_score": 52.1, "raw_scale": 100, "source_record": "Gemini 1.5 Pro (Feb 2024)|LMM 🖼️", "source_date": "2024-02-15" }, { "id": "88ec12c221a9ceec2eab", "model": "Claude 3 Opus", "organization": "Anthropic", "configuration": "LMM 🖼️", "score": 0.505, "raw_score": 50.5, "raw_scale": 100, "source_record": "Claude 3 Opus|LMM 🖼️", "source_date": "2024-03-04" }, { "id": "c14d3639f90c22f8bfb3", "model": "GPT-4V (Playground)", "organization": "OpenAI", "configuration": "LMM 🖼️", "score": 0.499, "raw_score": 49.9, "raw_scale": 100, "source_record": "GPT-4V (Playground)|LMM 🖼️", "source_date": "2023-10-15" }, { "id": "65635336712d27ecf921", "model": "Claude 3 Sonnet", "organization": "Anthropic", "configuration": "LMM 🖼️", "score": 0.479, "raw_score": 47.9, "raw_scale": 100, "source_record": "Claude 3 Sonnet|LMM 🖼️", "source_date": "2024-03-04" }, { "id": "81f4a331e5c2bc58dd2d", "model": "InternVL-Chat-V1.2", "organization": "", "configuration": "LMM 🖼️", "score": 0.47700000000000004, "raw_score": 47.7, "raw_scale": 100, "source_record": "InternVL-Chat-V1.2|LMM 🖼️", "source_date": "2024-02-22" }, { "id": "3e2edac0ef81308a5085", "model": "Math-LLaVA-13B", "organization": "", "configuration": "LMM 🖼️", "score": 0.466, "raw_score": 46.6, "raw_scale": 100, "source_record": "Math-LLaVA-13B|LMM 🖼️", "source_date": "2024-06-25" }, { "id": "1285520d46b43f29f68f", "model": "LLaVA-NeXT-34B", "organization": "", "configuration": "LMM 🖼️", "score": 0.465, "raw_score": 46.5, "raw_scale": 100, "source_record": "LLaVA-NeXT-34B|LMM 🖼️", "source_date": "2024-01-30" }, { "id": "c13bbdfe535f58978eaf", "model": "Claude 3 Haiku", "organization": "Anthropic", "configuration": "LMM 🖼️", "score": 0.46399999999999997, "raw_score": 46.4, "raw_scale": 100, "source_record": "Claude 3 Haiku|LMM 🖼️", "source_date": "2024-03-04" }, { "id": "f1f5a3546a2e0871e2d1", "model": "Gemini 1.0 Pro", "organization": "Google", "configuration": "LMM 🖼️", "score": 0.452, "raw_score": 45.2, "raw_scale": 100, "source_record": "Gemini 1.0 Pro|LMM 🖼️", "source_date": "2023-12-06" }, { "id": "30a0bf40de08f1a32c9d", "model": "Phi-3-Vision-128K-In", "organization": "Microsoft", "configuration": "LMM 🖼️", "score": 0.445, "raw_score": 44.5, "raw_scale": 100, "source_record": "Phi-3-Vision-128K-In|LMM 🖼️", "source_date": "2024-05-21" }, { "id": "88bbf8db81254703bded", "model": "Phi-3.5-Vision 4.2B", "organization": "Microsoft", "configuration": "LMM 🖼️", "score": 0.439, "raw_score": 43.9, "raw_scale": 100, "source_record": "Phi-3.5-Vision 4.2B|LMM 🖼️", "source_date": "2024-04-22" }, { "id": "acbd521a3c95245e3522", "model": "Mini-Gemini-HD (Hermes-2-Yi-34B)", "organization": "", "configuration": "LMM 🖼️", "score": 0.433, "raw_score": 43.3, "raw_scale": 100, "source_record": "Mini-Gemini-HD (Hermes-2-Yi-34B)|LMM 🖼️", "source_date": "2024-03-27" }, { "id": "cbf25fe2e62af37fb0f1", "model": "Qwen-VL-Plus", "organization": "Alibaba", "configuration": "LMM 🖼️", "score": 0.433, "raw_score": 43.3, "raw_scale": 100, "source_record": "Qwen-VL-Plus|LMM 🖼️", "source_date": "2023-12-21" }, { "id": "ff171e78fb6589864239", "model": "SPHINX-MoE", "organization": "", "configuration": "MoE 🤖", "score": 0.423, "raw_score": 42.3, "raw_scale": 100, "source_record": "SPHINX-MoE|MoE 🤖", "source_date": "2024-01-12" }, { "id": "4d3bcfa6e009473e2925", "model": "Mini-Gemini (Mixtral-8x7B)", "organization": "", "configuration": "LMM 🖼️", "score": 0.418, "raw_score": 41.8, "raw_scale": 100, "source_record": "Mini-Gemini (Mixtral-8x7B)|LMM 🖼️", "source_date": "2024-03-27" }, { "id": "8fe185b22a75088ab43f", "model": "MM1-7B-MoE-Chat", "organization": "", "configuration": "LMM 🖼️", "score": 0.409, "raw_score": 40.9, "raw_scale": 100, "source_record": "MM1-7B-MoE-Chat|LMM 🖼️", "source_date": "2024-03-14" }, { "id": "63cb54841bbaf1630298", "model": "MiniCPM-V-2 (2.8B)", "organization": "", "configuration": "LMM 🖼️", "score": 0.406, "raw_score": 40.6, "raw_scale": 100, "source_record": "MiniCPM-V-2 (2.8B)|LMM 🖼️", "source_date": "2024-04-14" }, { "id": "4e139cd09f9e36a6cdd2", "model": "MM1-30B-Chat", "organization": "", "configuration": "LMM 🖼️", "score": 0.39399999999999996, "raw_score": 39.4, "raw_scale": 100, "source_record": "MM1-30B-Chat|LMM 🖼️", "source_date": "2024-03-14" }, { "id": "2d384500a38d1d8064c7", "model": "SPHINX-Plus", "organization": "", "configuration": "MoE 🤖", "score": 0.368, "raw_score": 36.8, "raw_scale": 100, "source_record": "SPHINX-Plus|MoE 🤖", "source_date": "2024-01-12" }, { "id": "63e1388e760322709187", "model": "SPHINX (V2)", "organization": "", "configuration": "LMM 🖼️", "score": 0.36700000000000005, "raw_score": 36.7, "raw_scale": 100, "source_record": "SPHINX (V2)|LMM 🖼️", "source_date": "2023-11-17" }, { "id": "9be7d4f66cb175dca586", "model": "MM1-7B-Chat", "organization": "", "configuration": "LMM 🖼️", "score": 0.359, "raw_score": 35.9, "raw_scale": 100, "source_record": "MM1-7B-Chat|LMM 🖼️", "source_date": "2024-03-14" }, { "id": "944390c28b29095df166", "model": "SPHINX-Intern2", "organization": "", "configuration": "MoE 🤖", "score": 0.355, "raw_score": 35.5, "raw_scale": 100, "source_record": "SPHINX-Intern2|MoE 🤖", "source_date": "2024-01-12" }, { "id": "87d53b183cde42345ace", "model": "OmniLMM-12B", "organization": "", "configuration": "LMM 🖼️", "score": 0.349, "raw_score": 34.9, "raw_scale": 100, "source_record": "OmniLMM-12B|LMM 🖼️", "source_date": "2024-02-01" }, { "id": "b3301e383a5fd3d5e074", "model": "Multimodal Bard", "organization": "", "configuration": "LMM 🖼️", "score": 0.348, "raw_score": 34.8, "raw_scale": 100, "source_record": "Multimodal Bard|LMM 🖼️", "source_date": "2023-10-03" }, { "id": "35728c6086c27102677c", "model": "LLaVA-NeXT-Vicuna-7B", "organization": "", "configuration": "LMM 🖼️", "score": 0.34600000000000003, "raw_score": 34.6, "raw_scale": 100, "source_record": "LLaVA-NeXT-Vicuna-7B|LMM 🖼️", "source_date": "2024-01-30" }, { "id": "8548535f5c60ee2167a2", "model": "PoT GPT-4 (Caption+OCR)", "organization": "", "configuration": "Tool 🛠️", "score": 0.33899999999999997, "raw_score": 33.9, "raw_scale": 100, "source_record": "PoT GPT-4 (Caption+OCR)|Tool 🛠️", "source_date": "2023-10-03" }, { "id": "4016c9d48bd98d293b3e", "model": "CoT ChatGPT (Caption+OCR)", "organization": "", "configuration": "Tool 🛠️", "score": 0.332, "raw_score": 33.2, "raw_scale": 100, "source_record": "CoT ChatGPT (Caption+OCR)|Tool 🛠️", "source_date": "2023-10-03" }, { "id": "3880545f2bf34af98f5e", "model": "CoT Claude (Caption+OCR)", "organization": "", "configuration": "Tool 🛠️", "score": 0.332, "raw_score": 33.2, "raw_scale": 100, "source_record": "CoT Claude (Caption+OCR)|Tool 🛠️", "source_date": "2023-10-03" }, { "id": "ea186876d5febbf41383", "model": "CoT GPT4 (Caption+OCR)", "organization": "", "configuration": "Tool 🛠️", "score": 0.332, "raw_score": 33.2, "raw_scale": 100, "source_record": "CoT GPT4 (Caption+OCR)|Tool 🛠️", "source_date": "2023-10-03" }, { "id": "c5897fe954b8766279b0", "model": "MM1-3B-MoE-Chat", "organization": "", "configuration": "LMM 🖼️", "score": 0.326, "raw_score": 32.6, "raw_scale": 100, "source_record": "MM1-3B-MoE-Chat|LMM 🖼️", "source_date": "2024-03-14" }, { "id": "4cc8c8f4e42b67865327", "model": "MM1-3B-Chat", "organization": "", "configuration": "LMM 🖼️", "score": 0.32, "raw_score": 32.0, "raw_scale": 100, "source_record": "MM1-3B-Chat|LMM 🖼️", "source_date": "2024-03-14" }, { "id": "cd027dbe4c13460c3831", "model": "Gemini 1.0 Nano 2", "organization": "Google", "configuration": "LMM 🖼️", "score": 0.306, "raw_score": 30.6, "raw_scale": 100, "source_record": "Gemini 1.0 Nano 2|LMM 🖼️", "source_date": "2023-12-06" }, { "id": "068c64471e0eac17d678", "model": "LLaVA-1.5-13B", "organization": "", "configuration": "LMM 🖼️", "score": 0.276, "raw_score": 27.6, "raw_scale": 100, "source_record": "LLaVA-1.5-13B|LMM 🖼️", "source_date": "2024-01-30" }, { "id": "c03b633ee44c689c250c", "model": "SPHINX (V1)", "organization": "", "configuration": "LMM 🖼️", "score": 0.275, "raw_score": 27.5, "raw_scale": 100, "source_record": "SPHINX (V1)|LMM 🖼️", "source_date": "2023-11-09" }, { "id": "8459e793063f5311ad06", "model": "Gemini 1.0 Nano 1", "organization": "Google", "configuration": "LMM 🖼️", "score": 0.273, "raw_score": 27.3, "raw_scale": 100, "source_record": "Gemini 1.0 Nano 1|LMM 🖼️", "source_date": "2023-12-06" }, { "id": "aff3bd6d1c2a945d8994", "model": "PoT ChatGPT (Caption+OCR)", "organization": "", "configuration": "Tool 🛠️", "score": 0.268, "raw_score": 26.8, "raw_scale": 100, "source_record": "PoT ChatGPT (Caption+OCR)|Tool 🛠️", "source_date": "2023-10-03" }, { "id": "4068f74be94392296e80", "model": "SPHINX-Tiny", "organization": "", "configuration": "MoE 🤖", "score": 0.264, "raw_score": 26.4, "raw_scale": 100, "source_record": "SPHINX-Tiny|MoE 🤖", "source_date": "2024-01-12" }, { "id": "a5690f0504cad145d146", "model": "LLaVA (LLaMA-2-13B)", "organization": "", "configuration": "LMM 🖼️", "score": 0.261, "raw_score": 26.1, "raw_scale": 100, "source_record": "LLaVA (LLaMA-2-13B)|LMM 🖼️", "source_date": "2023-10-03" }, { "id": "e31396f9778dc9f93fce", "model": "InstructBLIP (Vicuna-7B)", "organization": "", "configuration": "LMM 🖼️", "score": 0.253, "raw_score": 25.3, "raw_scale": 100, "source_record": "InstructBLIP (Vicuna-7B)|LMM 🖼️", "source_date": "2023-10-03" }, { "id": "bd8a4fd3553fe9db8c1a", "model": "LLaVAR", "organization": "", "configuration": "LMM 🖼️", "score": 0.252, "raw_score": 25.2, "raw_scale": 100, "source_record": "LLaVAR|LMM 🖼️", "source_date": "2023-10-03" }, { "id": "26c0562a3e4f45d0e597", "model": "LLaMA-Adapter-V2 (7B)", "organization": "Meta", "configuration": "LMM 🖼️", "score": 0.239, "raw_score": 23.9, "raw_scale": 100, "source_record": "LLaMA-Adapter-V2 (7B)|LMM 🖼️", "source_date": "2023-10-03" }, { "id": "86d9d44ddd80a86a6433", "model": "miniGPT4 (LLaMA-2-7B)", "organization": "", "configuration": "LMM 🖼️", "score": 0.231, "raw_score": 23.1, "raw_scale": 100, "source_record": "miniGPT4 (LLaMA-2-7B)|LMM 🖼️", "source_date": "2023-10-03" }, { "id": "4efc6cd476cbd7b6fc30", "model": "mPLUG-Owl (LLaMA-7B)", "organization": "", "configuration": "LMM 🖼️", "score": 0.222, "raw_score": 22.2, "raw_scale": 100, "source_record": "mPLUG-Owl (LLaMA-7B)|LMM 🖼️", "source_date": "2023-10-03" }, { "id": "896858e9fe84432bc70d", "model": "IDEFICS (9B-Instruct)", "organization": "", "configuration": "LMM 🖼️", "score": 0.198, "raw_score": 19.8, "raw_scale": 100, "source_record": "IDEFICS (9B-Instruct)|LMM 🖼️", "source_date": "2023-10-03" } ] }, { "id": "math-vision-full", "name": "MATH-Vision (full test)", "group": "数学与科学", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Full 3,040-question test set. Official board includes publisher and third-party submissions; tool use and third-party labels are retained in result names.", "zh": "完整 3,040 道测试题。官方榜单包含模型发布方与第三方提交；结果名称保留工具使用和第三方标注。" }, "source": { "data_url": "https://mathllm.github.io/mathvision/", "sha256": "4f50613509b976da1a6197298a180f02064fd5d2d842b757c1277c8064e95467", "retrieved_at": "2026-11-01T16:38:11.408410+00:00", "name": "MATH-Vision", "url": "https://mathllm.github.io/mathvision/", "selector": "First #results table: ALL; not testmini or Wild; omit human/random", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "17eccd0a2aab79b8a49c", "model": "GPT-5.4 (xhigh reasoning, w/ Python) (3rd-party eval)", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.961, "raw_score": 96.1, "raw_scale": 100, "source_record": "GPT-5.4 (xhigh reasoning, w/ Python) (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-01", "report_url": "https://www.kimi.com/blog/kimi-k2-6" }, { "id": "e7af0a1c86b73ab31bc3", "model": "Gemini 3.1 Pro (thinking high, w/ Python) (3rd-party eval)", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.9570000000000001, "raw_score": 95.7, "raw_scale": 100, "source_record": "Gemini 3.1 Pro (thinking high, w/ Python) (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-01", "report_url": "https://www.kimi.com/blog/kimi-k2-6" }, { "id": "d4efc9db1ecf4526d071", "model": "Kimi K2.6 (w/ Python)", "organization": "Moonshot AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.932, "raw_score": 93.2, "raw_scale": 100, "source_record": "Kimi K2.6 (w/ Python)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-01", "report_url": "https://www.kimi.com/blog/kimi-k2-6" }, { "id": "e9285da269d16c31a9f9", "model": "GPT-5.4 (xhigh reasoning) (3rd-party eval)", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.92, "raw_score": 92.0, "raw_scale": 100, "source_record": "GPT-5.4 (xhigh reasoning) (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-01", "report_url": "https://www.kimi.com/blog/kimi-k2-6" }, { "id": "69d8149bdd853c3296d2", "model": "Gemini 3.0 Flash (3rd-party eval)", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.9059999999999999, "raw_score": 90.6, "raw_scale": 100, "source_record": "Gemini 3.0 Flash (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-28", "report_url": "https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seed-2-0-lite" }, { "id": "1aaa749f74aac969b1d2", "model": "Doubao-Seed-2.0-lite-0428", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.898, "raw_score": 89.8, "raw_scale": 100, "source_record": "Doubao-Seed-2.0-lite-0428|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-28", "report_url": "https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seed-2-0-lite" }, { "id": "61630a59e900f51c8859", "model": "Gemini 3.1 Pro (thinking high) (3rd-party eval)", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.898, "raw_score": 89.8, "raw_scale": 100, "source_record": "Gemini 3.1 Pro (thinking high) (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-01", "report_url": "https://www.kimi.com/blog/kimi-k2-6" }, { "id": "278e0511127d109c6076", "model": "Doubao-Seed-2.0-pro-0215", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.888, "raw_score": 88.8, "raw_scale": 100, "source_record": "Doubao-Seed-2.0-pro-0215|Full test set (3,040 problems); settings in result name", "source_date": "2026-02-15", "report_url": "https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seed-2-0-lite" }, { "id": "842c142f580af25b19fb", "model": "Qwen3.5-397B-A17B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.8859999999999999, "raw_score": 88.6, "raw_scale": 100, "source_record": "Qwen3.5-397B-A17B|Full test set (3,040 problems); settings in result name", "source_date": "2026-02-17", "report_url": "https://qwen.ai/blog?id=qwen3.5" }, { "id": "df3567c42a14891aba76", "model": "Kimi K2.6 (no tools)", "organization": "Moonshot AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.8740000000000001, "raw_score": 87.4, "raw_scale": 100, "source_record": "Kimi K2.6 (no tools)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-01", "report_url": "https://www.kimi.com/blog/kimi-k2-6" }, { "id": "c1a6d506e4b18079b80a", "model": "Gemini 3 Pro (3rd-party eval)", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.866, "raw_score": 86.6, "raw_scale": 100, "source_record": "Gemini 3 Pro (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-02-17", "report_url": "https://huggingface.co/blog/mlabonne/qwen35" }, { "id": "f7e0134fe54d050cb383", "model": "Doubao-Seed-2.0-lite-0215", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.8640000000000001, "raw_score": 86.4, "raw_scale": 100, "source_record": "Doubao-Seed-2.0-lite-0215|Full test set (3,040 problems); settings in result name", "source_date": "2026-02-15", "report_url": "https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seed-2-0-lite" }, { "id": "7048d87c4e4b0a9284f3", "model": "Qwen3.5-122B-A10B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.862, "raw_score": 86.2, "raw_scale": 100, "source_record": "Qwen3.5-122B-A10B|Full test set (3,040 problems); settings in result name", "source_date": "2026-02-16", "report_url": "https://huggingface.co/Qwen/Qwen3.5-27B" }, { "id": "16ff8b446f85271629ca", "model": "Qwen3.5-27B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.86, "raw_score": 86.0, "raw_scale": 100, "source_record": "Qwen3.5-27B|Full test set (3,040 problems); settings in result name", "source_date": "2026-02-16", "report_url": "https://huggingface.co/Qwen/Qwen3.5-27B" }, { "id": "52bb39de682ca9d248a2", "model": "Gemma 4 31B", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.856, "raw_score": 85.6, "raw_scale": 100, "source_record": "Gemma 4 31B|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-02", "report_url": "https://huggingface.co/blog/gemma4" }, { "id": "f342b8bb72521c7a2bf7", "model": "Kimi K2.5 (thinking, w/ Python)", "organization": "Moonshot AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.85, "raw_score": 85.0, "raw_scale": 100, "source_record": "Kimi K2.5 (thinking, w/ Python)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-01", "report_url": "https://www.kimi.com/blog/kimi-k2-6" }, { "id": "37fe5e4bb4ca7ab52a69", "model": "Claude Opus 4.6 (max effort, w/ Python) (3rd-party eval)", "organization": "Anthropic", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.846, "raw_score": 84.6, "raw_scale": 100, "source_record": "Claude Opus 4.6 (max effort, w/ Python) (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-01", "report_url": "https://www.kimi.com/blog/kimi-k2-6" }, { "id": "0bc569a6e02c14d5f3da", "model": "Kimi K2.5", "organization": "Moonshot AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.8420000000000001, "raw_score": 84.2, "raw_scale": 100, "source_record": "Kimi K2.5|Full test set (3,040 problems); settings in result name", "source_date": "2026-01-27", "report_url": "https://build.nvidia.com/moonshotai/kimi-k2.5/modelcard" }, { "id": "6179a13894ca03828121", "model": "Qwen3.5-35B-A3B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.8390000000000001, "raw_score": 83.9, "raw_scale": 100, "source_record": "Qwen3.5-35B-A3B|Full test set (3,040 problems); settings in result name", "source_date": "2026-02-16", "report_url": "https://huggingface.co/Qwen/Qwen3.5-27B" }, { "id": "b9d8478c2c8adf44f6f6", "model": "GPT-5.2 (3rd-party eval)", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.83, "raw_score": 83.0, "raw_scale": 100, "source_record": "GPT-5.2 (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-01-27", "report_url": "https://arxiv.org/html/2602.02276v1" }, { "id": "0cf2cec1ee0341e58119", "model": "Gemma 4 26B A4B", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.8240000000000001, "raw_score": 82.4, "raw_scale": 100, "source_record": "Gemma 4 26B A4B|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-02", "report_url": "https://huggingface.co/blog/gemma4" }, { "id": "4954042936c497b4e105", "model": "Seed-1.8", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.813, "raw_score": 81.3, "raw_scale": 100, "source_record": "Seed-1.8|Full test set (3,040 problems); settings in result name", "source_date": "2026-03-01", "report_url": "https://lf3-static.bytednsdoc.com/obj/eden-cn/lapzild-tss/ljhwZthlaukjlkulzlp/research/Seed-1.8-Modelcard.pdf" }, { "id": "15d7c00daa59f6891014", "model": "Seed 1.6-Thinking", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.772, "raw_score": 77.2, "raw_scale": 100, "source_record": "Seed 1.6-Thinking|Full test set (3,040 problems); settings in result name", "source_date": "2025-06-25", "report_url": "https://lf3-static.bytednsdoc.com/obj/eden-cn/lapzild-tss/ljhwZthlaukjlkulzlp/research/Seed-1.8-Modelcard.pdf" }, { "id": "b1f960bb18cbbb96adfc", "model": "Claude Opus 4.5 (3rd-party eval)", "organization": "Anthropic", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.7709999999999999, "raw_score": 77.1, "raw_scale": 100, "source_record": "Claude Opus 4.5 (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-01-27", "report_url": "https://arxiv.org/html/2602.02276v1" }, { "id": "bd5c9c2709e901803756", "model": "Step3-VL-10B (PaCoRe)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.7595000000000001, "raw_score": 75.95, "raw_scale": 100, "source_record": "Step3-VL-10B (PaCoRe)|Full test set (3,040 problems); settings in result name", "source_date": "2026-01-15", "report_url": "https://arxiv.org/abs/2601.09668" }, { "id": "4679f5cf9bd2fef04f59", "model": "Qwen3-VL-235B-A22B-Thinking", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.746, "raw_score": 74.6, "raw_scale": 100, "source_record": "Qwen3-VL-235B-A22B-Thinking|Full test set (3,040 problems); settings in result name", "source_date": "2025-09-20", "report_url": "https://arxiv.org/abs/2511.21631" }, { "id": "cf11a4dc24989cdadf34", "model": "Gemini 2.5 Pro", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.733, "raw_score": 73.3, "raw_scale": 100, "source_record": "Gemini 2.5 Pro|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-23", "report_url": "https://deepmind.google/technologies/gemini/pro/" }, { "id": "792b7d7ea223858d28a3", "model": "GPT-5", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "GPT-5|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-26", "report_url": "https://openai.com/gpt-5/" }, { "id": "4db293958e2bb427f3f2", "model": "GPT-5-mini (2025-08-07) (3rd-party eval)", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.7190000000000001, "raw_score": 71.9, "raw_scale": 100, "source_record": "GPT-5-mini (2025-08-07) (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-02-16", "report_url": "https://huggingface.co/Qwen/Qwen3.5-27B" }, { "id": "f68e921bbbcae165a8f0", "model": "Claude Opus 4.6 (max effort) (3rd-party eval)", "organization": "Anthropic", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.7120000000000001, "raw_score": 71.2, "raw_scale": 100, "source_record": "Claude Opus 4.6 (max effort) (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-01", "report_url": "https://www.kimi.com/blog/kimi-k2-6" }, { "id": "accfc9d15875889c6aaa", "model": "Claude Sonnet 4.5 (3rd-party eval)", "organization": "Anthropic", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.711, "raw_score": 71.1, "raw_scale": 100, "source_record": "Claude Sonnet 4.5 (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2026-02-16", "report_url": "https://huggingface.co/Qwen/Qwen3.5-27B" }, { "id": "9748f1f08b8371a6ac1a", "model": "Step3-VL-10B (SeRe)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.7081000000000001, "raw_score": 70.81, "raw_scale": 100, "source_record": "Step3-VL-10B (SeRe)|Full test set (3,040 problems); settings in result name", "source_date": "2026-01-15", "report_url": "https://arxiv.org/abs/2601.09668" }, { "id": "7307969a9fef37aea227", "model": "dots.vlm1", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.6964, "raw_score": 69.64, "raw_scale": 100, "source_record": "dots.vlm1|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-05", "report_url": "https://github.com/rednote-hilab/dots.vlm1" }, { "id": "2d0f7bee2306b20b6da2", "model": "Seed1.5-VL", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.687, "raw_score": 68.7, "raw_scale": 100, "source_record": "Seed1.5-VL|Full test set (3,040 problems); settings in result name", "source_date": "2025-05-12", "report_url": "https://github.com/ByteDance-Seed/Seed1.5-VL/" }, { "id": "6d8976280e87ccc57ea0", "model": "Claude Opus 4.1 (thinking) (3rd-party eval)", "organization": "Anthropic", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.66, "raw_score": 66.0, "raw_scale": 100, "source_record": "Claude Opus 4.1 (thinking) (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-05", "report_url": "https://arxiv.org/abs/2511.21631" }, { "id": "12f34a2583a608824664", "model": "Qwen3-VL-235B-A22B-Instruct", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.66, "raw_score": 66.0, "raw_scale": 100, "source_record": "Qwen3-VL-235B-A22B-Instruct|Full test set (3,040 problems); settings in result name", "source_date": "2025-09-20", "report_url": "https://arxiv.org/abs/2511.21631" }, { "id": "e94295f6a3d7f327dbc4", "model": "GLM-4.5V (106B-A12B)", "organization": "Z AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.6559999999999999, "raw_score": 65.6, "raw_scale": 100, "source_record": "GLM-4.5V (106B-A12B)|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-26", "report_url": "https://huggingface.co/zai-org/GLM-4.5V" }, { "id": "52d8ddbede5e8ab2e641", "model": "Step-3 (321B-A38B)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.648, "raw_score": 64.8, "raw_scale": 100, "source_record": "Step-3 (321B-A38B)|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-26", "report_url": "https://stepfun.ai/research/en/step3/" }, { "id": "87cf9a5900f61f28beae", "model": "InternVL3.5 (241B-A28B)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.639, "raw_score": 63.9, "raw_scale": 100, "source_record": "InternVL3.5 (241B-A28B)|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-26", "report_url": "https://huggingface.co/OpenGVLab/InternVL3_5-241B-A28B" }, { "id": "37695166d1033ce4e487", "model": "GLM-4.6V (106B-A12B)", "organization": "Z AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.635, "raw_score": 63.5, "raw_scale": 100, "source_record": "GLM-4.6V (106B-A12B)|Full test set (3,040 problems); settings in result name", "source_date": "2025-11-01", "report_url": "https://arxiv.org/abs/2507.01006" }, { "id": "10520ea988bac3c6ab72", "model": "MiMo-VL-RL", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.604, "raw_score": 60.4, "raw_scale": 100, "source_record": "MiMo-VL-RL|Full test set (3,040 problems); settings in result name", "source_date": "2025-06-04", "report_url": "https://github.com/XiaomiMiMo/MiMo-VL/" }, { "id": "e6a50d33ad3b8ebd1a7c", "model": "OpenAI o1", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.603, "raw_score": 60.3, "raw_scale": 100, "source_record": "OpenAI o1|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-10", "report_url": "https://openai.com/o1/" }, { "id": "a89c0b81d6496d1f719c", "model": "MiMo-VL-RL-2508", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.5965, "raw_score": 59.65, "raw_scale": 100, "source_record": "MiMo-VL-RL-2508|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-01", "report_url": "https://arxiv.org/abs/2506.03569" }, { "id": "ac8b4d347333a6fcf9d7", "model": "Qwen3-VL-8B-Thinking", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.596, "raw_score": 59.6, "raw_scale": 100, "source_record": "Qwen3-VL-8B-Thinking|Full test set (3,040 problems); settings in result name", "source_date": "2025-10-01", "report_url": "https://arxiv.org/abs/2511.21631" }, { "id": "bb4c18c0c58b30562779", "model": "Gemma 4 E4B", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.595, "raw_score": 59.5, "raw_scale": 100, "source_record": "Gemma 4 E4B|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-02", "report_url": "https://huggingface.co/blog/gemma4" }, { "id": "8b24bee25b0bec0120b0", "model": "Claude 3.7 Sonnet (3rd-party eval, Skywork)", "organization": "Anthropic", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.586, "raw_score": 58.6, "raw_scale": 100, "source_record": "Claude 3.7 Sonnet (3rd-party eval, Skywork)|Full test set (3,040 problems); settings in result name", "source_date": "2025-02-24", "report_url": "https://huggingface.co/Skywork/Skywork-R1V3-38B" }, { "id": "41c11f5e5e3869ea1354", "model": "OpenAI o4-mini (3rd-party eval)", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.58, "raw_score": 58.0, "raw_scale": 100, "source_record": "OpenAI o4-mini (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-16", "report_url": "https://huggingface.co/Skywork/Skywork-R1V2-38B-AWQ" }, { "id": "06327e348768f5c7403d", "model": "MiMo-VL-7B-SFT", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.579, "raw_score": 57.9, "raw_scale": 100, "source_record": "MiMo-VL-7B-SFT|Full test set (3,040 problems); settings in result name", "source_date": "2025-06-04", "report_url": "https://arxiv.org/abs/2506.03569" }, { "id": "97484788b15389768731", "model": "Kimi-VL-A3B-Thinking-2506", "organization": "Moonshot AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.569, "raw_score": 56.9, "raw_scale": 100, "source_record": "Kimi-VL-A3B-Thinking-2506|Full test set (3,040 problems); settings in result name", "source_date": "2025-06-23", "report_url": "https://arxiv.org/pdf/2504.07491" }, { "id": "a46b1789e1d703b521d9", "model": "Step R1-V-Mini", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.5660000000000001, "raw_score": 56.6, "raw_scale": 100, "source_record": "Step R1-V-Mini|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-05", "report_url": "https://platform.stepfun.com" }, { "id": "5d5b33f32b3fcc16e5ce", "model": "InternVL3.5 (30B-A3B)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.557, "raw_score": 55.7, "raw_scale": 100, "source_record": "InternVL3.5 (30B-A3B)|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-26", "report_url": "https://huggingface.co/OpenGVLab/InternVL3_5-30B-A3B" }, { "id": "168059290f668529dbd6", "model": "SenseNova V6 Reasoner", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.5539000000000001, "raw_score": 55.39, "raw_scale": 100, "source_record": "SenseNova V6 Reasoner|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-10", "report_url": "https://platform.sensenova.cn/" }, { "id": "4e809e2c78605fc0faac", "model": "GLM-4.1V (9B)", "organization": "Z AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.544, "raw_score": 54.4, "raw_scale": 100, "source_record": "GLM-4.1V (9B)|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-26", "report_url": "https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking" }, { "id": "9fa7864caf4d94e73271", "model": "GLM-4.6V-Flash (9B)", "organization": "Z AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.5405, "raw_score": 54.05, "raw_scale": 100, "source_record": "GLM-4.6V-Flash (9B)|Full test set (3,040 problems); settings in result name", "source_date": "2025-11-01", "report_url": "https://arxiv.org/abs/2507.01006" }, { "id": "acae868a54a1ed9ed589", "model": "InternVL3.5-38B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.54, "raw_score": 54.0, "raw_scale": 100, "source_record": "InternVL3.5-38B|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-26", "report_url": "https://arxiv.org/html/2508.18265v1" }, { "id": "f25cae5a39f2cb187963", "model": "AStar-7B (training-free, Qwen2.5-VL-7B)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.539, "raw_score": 53.9, "raw_scale": 100, "source_record": "AStar-7B (training-free, Qwen2.5-VL-7B)|Full test set (3,040 problems); settings in result name", "source_date": "2025-02-04", "report_url": "https://arxiv.org/abs/2502.02339" }, { "id": "e93a6b9f59438a54e649", "model": "Ovis2.5-9B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.539, "raw_score": 53.9, "raw_scale": 100, "source_record": "Ovis2.5-9B|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-19", "report_url": "https://arxiv.org/abs/2508.11737" }, { "id": "6058acca71ad76486caf", "model": "Kimi k1.6 Preview", "organization": "Moonshot AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.5329, "raw_score": 53.29, "raw_scale": 100, "source_record": "Kimi k1.6 Preview|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-08", "report_url": "https://github.com/MoonshotAI/Kimi-k1.5" }, { "id": "3f195628c9ca6fe3410a", "model": "Decoupled LLM-LMM (Qwen2.5-VL-72B + Qwen3-32B)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.526, "raw_score": 52.6, "raw_scale": 100, "source_record": "Decoupled LLM-LMM (Qwen2.5-VL-72B + Qwen3-32B)|Full test set (3,040 problems); settings in result name", "source_date": "2025-09-27", "report_url": "https://arxiv.org/abs/2509.23322" }, { "id": "36ecec53802154d72a1d", "model": "Skywork-R1V3-38B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.526, "raw_score": 52.6, "raw_scale": 100, "source_record": "Skywork-R1V3-38B|Full test set (3,040 problems); settings in result name", "source_date": "2025-07-05", "report_url": "https://huggingface.co/Skywork/Skywork-R1V3-38B" }, { "id": "f718255f26912c2101ab", "model": "Gemma 4 E2B", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.524, "raw_score": 52.4, "raw_scale": 100, "source_record": "Gemma 4 E2B|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-02", "report_url": "https://huggingface.co/blog/gemma4" }, { "id": "e5c17cbb84a527db20af", "model": "InternVL3.5-8B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.5205, "raw_score": 52.05, "raw_scale": 100, "source_record": "InternVL3.5-8B|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-26", "report_url": "https://arxiv.org/html/2508.18265v1" }, { "id": "bd7558c4d5cca9415990", "model": "Open-Vision-Reasoner-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.518, "raw_score": 51.8, "raw_scale": 100, "source_record": "Open-Vision-Reasoner-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-07-15", "report_url": "https://arxiv.org/abs/2507.05255" }, { "id": "d2092eb1f90b024d70cd", "model": "Qianfan-VL-70B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.5029, "raw_score": 50.29, "raw_scale": 100, "source_record": "Qianfan-VL-70B|Full test set (3,040 problems); settings in result name", "source_date": "2025-09-16", "report_url": "https://baidubce.github.io/Qianfan-VL" }, { "id": "d380539c17f8a99b5b81", "model": "Skywork-R1V2-38B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.49700000000000005, "raw_score": 49.7, "raw_scale": 100, "source_record": "Skywork-R1V2-38B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-28", "report_url": "https://github.com/SkyworkAI/Skywork-R1V" }, { "id": "563f887f6eda1a007708", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.48619999999999997, "raw_score": 48.62, "raw_scale": 100, "source_record": "Doubao-1.5-pro|Full test set (3,040 problems); settings in result name", "source_date": "2025-02-28", "report_url": "https://team.doubao.com/zh/special/doubao_1_5_pro" }, { "id": "d7fdfdd118901df37831", "model": "Gemini 2.0 Pro (3rd-party eval)", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.48100000000000004, "raw_score": 48.1, "raw_scale": 100, "source_record": "Gemini 2.0 Pro (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2025-02-05", "report_url": "https://internvl.github.io/blog/2025-04-11-InternVL-3.0/" }, { "id": "ce91af406c7df4efa2d4", "model": "GPT-4.5", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.473, "raw_score": 47.3, "raw_scale": 100, "source_record": "GPT-4.5|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-10", "report_url": "https://openai.com/index/introducing-gpt-4-5/" }, { "id": "ad2cb2f3981b9075101d", "model": "Gemma 3 27B (no think)", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.46, "raw_score": 46.0, "raw_scale": 100, "source_record": "Gemma 3 27B (no think)|Full test set (3,040 problems); settings in result name", "source_date": "2026-04-02", "report_url": "https://huggingface.co/blog/gemma4" }, { "id": "c68ba0fef422a19b53a5", "model": "Keye-VL-8B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.46, "raw_score": 46.0, "raw_scale": 100, "source_record": "Keye-VL-8B|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-19", "report_url": "https://arxiv.org/abs/2508.11737" }, { "id": "f9d06068ee0229a23ad9", "model": "GPT-5 (minimal) (3rd-party eval)", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.45799999999999996, "raw_score": 45.8, "raw_scale": 100, "source_record": "GPT-5 (minimal) (3rd-party eval)|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-07", "report_url": "https://arxiv.org/abs/2511.21631" }, { "id": "5306f595423ee39ad159", "model": "VL-Rethinker-72B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.4493, "raw_score": 44.93, "raw_scale": 100, "source_record": "VL-Rethinker-72B|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-25", "report_url": "https://tiger-ai-lab.github.io/VL-Rethinker/" }, { "id": "d2d2b591bf9efa4aacb5", "model": "LLaVA-Critic-R1 (Qwen2.5-VL-7B base)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.441, "raw_score": 44.1, "raw_scale": 100, "source_record": "LLaVA-Critic-R1 (Qwen2.5-VL-7B base)|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-30", "report_url": "https://arxiv.org/abs/2509.00676" }, { "id": "0291623c20f4423ebbd9", "model": "Vision-R1-7B (contamination-flagged)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.439, "raw_score": 43.9, "raw_scale": 100, "source_record": "Vision-R1-7B (contamination-flagged)|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-10", "report_url": "https://arxiv.org/abs/2503.06749" }, { "id": "6444bc8cd06e4f6015e8", "model": "InternVL3-78B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.431, "raw_score": 43.1, "raw_scale": 100, "source_record": "InternVL3-78B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-11", "report_url": "https://arxiv.org/abs/2504.10479" }, { "id": "21f006653cc3ce2e79c9", "model": "Skywork-R1V2-38B-AWQ", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.429, "raw_score": 42.9, "raw_scale": 100, "source_record": "Skywork-R1V2-38B-AWQ|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-28", "report_url": "https://huggingface.co/Skywork/Skywork-R1V2-38B-AWQ" }, { "id": "d4493a6b34aefc93ecf1", "model": "INFRL-Qwen2.5-VL-72B-Preview", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.42729999999999996, "raw_score": 42.73, "raw_scale": 100, "source_record": "INFRL-Qwen2.5-VL-72B-Preview|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-25", "report_url": "https://huggingface.co/infly/INFRL-Qwen2.5-VL-72B-Preview" }, { "id": "7ae2d45b40edfb55f362", "model": "Gemini-2 Flash", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.413, "raw_score": 41.3, "raw_scale": 100, "source_record": "Gemini-2 Flash|Full test set (3,040 problems); settings in result name", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "80b5e742399025e111bf", "model": "ProxyThinker (Qwen2.5-VL-32B + OpenVLThinker-7B expert)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.408, "raw_score": 40.8, "raw_scale": 100, "source_record": "ProxyThinker (Qwen2.5-VL-32B + OpenVLThinker-7B expert)|Full test set (3,040 problems); settings in result name", "source_date": "2025-05-30", "report_url": "https://arxiv.org/abs/2505.24872" }, { "id": "56d77d92cb0f8a656f2b", "model": "VL-Rethinker-32B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "VL-Rethinker-32B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-12", "report_url": "https://arxiv.org/abs/2504.08837" }, { "id": "06047a38182ae78ae81b", "model": "ViCrit-RL-72B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.401, "raw_score": 40.1, "raw_scale": 100, "source_record": "ViCrit-RL-72B|Full test set (3,040 problems); settings in result name", "source_date": "2025-06-11", "report_url": "https://arxiv.org/abs/2506.10128" }, { "id": "59295952e0fbc8e6428d", "model": "InternVL3.5-4B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.4, "raw_score": 40.0, "raw_scale": 100, "source_record": "InternVL3.5-4B|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-26", "report_url": "https://arxiv.org/html/2508.18265v1" }, { "id": "f95d76e2056cfcb59949", "model": "Kimi k1.5", "organization": "Moonshot AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.386, "raw_score": 38.6, "raw_scale": 100, "source_record": "Kimi k1.5|Full test set (3,040 problems); settings in result name", "source_date": "2025-01-22", "report_url": "https://arxiv.org/pdf/2501.12599" }, { "id": "aec31cee8a2b9da0bfdc", "model": "Virgo-72B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.384, "raw_score": 38.4, "raw_scale": 100, "source_record": "Virgo-72B|Full test set (3,040 problems); settings in result name", "source_date": "2025-01-03", "report_url": "https://arxiv.org/abs/2501.01904" }, { "id": "6e466f0e1b0b4daf6d52", "model": "Qwen2.5-VL-72B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.381, "raw_score": 38.1, "raw_scale": 100, "source_record": "Qwen2.5-VL-72B|Full test set (3,040 problems); settings in result name", "source_date": "2025-01-26" }, { "id": "1017fdceb004c2e39a8d", "model": "Claude3.5-Sonnet", "organization": "Anthropic", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.3799, "raw_score": 37.99, "raw_scale": 100, "source_record": "Claude3.5-Sonnet|Full test set (3,040 problems); settings in result name", "source_date": "2024-06-21", "report_url": "https://www.anthropic.com/news/claude-3-5-sonnet/" }, { "id": "773d186f4f68679f450d", "model": "AVAR-Thinker-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.374, "raw_score": 37.4, "raw_scale": 100, "source_record": "AVAR-Thinker-7B|Full test set (3,040 problems); settings in result name", "source_date": "2026-03-05", "report_url": "https://arxiv.org/abs/2603.03825" }, { "id": "9799238141057b4d344b", "model": "Ovis2.5-2B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.374, "raw_score": 37.4, "raw_scale": 100, "source_record": "Ovis2.5-2B|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-19", "report_url": "https://arxiv.org/abs/2508.11737" }, { "id": "0777bf4bafc8ffa4704f", "model": "InternVL3-14B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.37, "raw_score": 37.0, "raw_scale": 100, "source_record": "InternVL3-14B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-11", "report_url": "https://arxiv.org/abs/2504.10479" }, { "id": "ca8f1fe45dbf962a1365", "model": "Kimi-VL-A3B-Thinking", "organization": "Moonshot AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.368, "raw_score": 36.8, "raw_scale": 100, "source_record": "Kimi-VL-A3B-Thinking|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-11", "report_url": "https://arxiv.org/abs/2504.07491" }, { "id": "284f9ba832f3c68468f1", "model": "Phi-4-reasoning-vision-15B (testmini)", "organization": "Microsoft", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.36200000000000004, "raw_score": 36.2, "raw_scale": 100, "source_record": "Phi-4-reasoning-vision-15B (testmini)|Full test set (3,040 problems); settings in result name", "source_date": "2026-03-06", "report_url": "https://huggingface.co/microsoft/Phi-4-reasoning-vision-15B" }, { "id": "2ad2fbf810fd21c9c192", "model": "QvQ-72B-Preview", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.359, "raw_score": 35.9, "raw_scale": 100, "source_record": "QvQ-72B-Preview|Full test set (3,040 problems); settings in result name", "source_date": "2024-12-25", "report_url": "https://qwenlm.github.io/blog/qvq-72b-preview/" }, { "id": "01fbd895c6d79ad5774a", "model": "URSA-8B + URSA-RM (BoN=32)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.35100000000000003, "raw_score": 35.1, "raw_scale": 100, "source_record": "URSA-8B + URSA-RM (BoN=32)|Full test set (3,040 problems); settings in result name", "source_date": "2025-01-08", "report_url": "https://arxiv.org/abs/2501.04686" }, { "id": "23f81f517897a4475c15", "model": "InternVL3-38B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "InternVL3-38B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-11", "report_url": "https://arxiv.org/abs/2504.10479" }, { "id": "48cb4307ce19f10c464e", "model": "ThinkLite-VL-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.32899999999999996, "raw_score": 32.9, "raw_scale": 100, "source_record": "ThinkLite-VL-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-10", "report_url": "https://arxiv.org/abs/2504.07934" }, { "id": "26e66ceaf3575f29ba2b", "model": "Qianfan-VL-8B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.3282, "raw_score": 32.82, "raw_scale": 100, "source_record": "Qianfan-VL-8B|Full test set (3,040 problems); settings in result name", "source_date": "2025-09-16", "report_url": "https://baidubce.github.io/Qianfan-VL" }, { "id": "8b85a6dfb8ea9d26e8e5", "model": "InternVL2.5-78B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.322, "raw_score": 32.2, "raw_scale": 100, "source_record": "InternVL2.5-78B|Full test set (3,040 problems); settings in result name", "source_date": "2024-12-05", "report_url": "https://huggingface.co/OpenGVLab/InternVL2_5-78B" }, { "id": "cc4dc09cf9ee277c6c7f", "model": "Ovis2-34B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.319, "raw_score": 31.9, "raw_scale": 100, "source_record": "Ovis2-34B|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-25", "report_url": "https://github.com/AIDC-AI/Ovis" }, { "id": "09002a3e81c551d0af87", "model": "InternVL2.5-38B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "InternVL2.5-38B|Full test set (3,040 problems); settings in result name", "source_date": "2024-12-05", "report_url": "https://huggingface.co/OpenGVLab/InternVL2_5-38B" }, { "id": "f0cb5b28bfe7d1b46491", "model": "URSA-8B-PS-GRPO", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.315, "raw_score": 31.5, "raw_scale": 100, "source_record": "URSA-8B-PS-GRPO|Full test set (3,040 problems); settings in result name", "source_date": "2025-05-01", "report_url": "https://arxiv.org/abs/2501.04686" }, { "id": "567e94a7c6285dc265c8", "model": "TBAC-VLR1-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.314, "raw_score": 31.4, "raw_scale": 100, "source_record": "TBAC-VLR1-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-09-03", "report_url": "https://arxiv.org/abs/2509.03321" }, { "id": "ce0eee3a6a48a5b9a817", "model": "LLaVA-Critic-R1 (LLaMA-3.2-11B-V base)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.309, "raw_score": 30.9, "raw_scale": 100, "source_record": "LLaVA-Critic-R1 (LLaMA-3.2-11B-V base)|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-30", "report_url": "https://arxiv.org/abs/2509.00676" }, { "id": "9359fbcda8566701fb6a", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.3039, "raw_score": 30.39, "raw_scale": 100, "source_record": "GPT-4o|Full test set (3,040 problems); settings in result name", "source_date": "2024-05-19", "report_url": "https://openai.com/index/hello-gpt-4o/" }, { "id": "9bf50447d33e49799da3", "model": "GPT-4 Turbo", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.30260000000000004, "raw_score": 30.26, "raw_scale": 100, "source_record": "GPT-4 Turbo|Full test set (3,040 problems); settings in result name", "source_date": "2024-05-19", "report_url": "https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo" }, { "id": "b0e099d6c11def5a5341", "model": "DualMindVLM-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.302, "raw_score": 30.2, "raw_scale": 100, "source_record": "DualMindVLM-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-11-20", "report_url": "https://arxiv.org/abs/2511.16670" }, { "id": "1f5fb8b16d44f2b1d184", "model": "MMR1-Math-v0-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.302, "raw_score": 30.2, "raw_scale": 100, "source_record": "MMR1-Math-v0-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-13", "report_url": "https://huggingface.co/MMR1/MMR1-Math-v0-7B" }, { "id": "f137578567d72d4cee28", "model": "Ovis2-16B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.301, "raw_score": 30.1, "raw_scale": 100, "source_record": "Ovis2-16B|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-25", "report_url": "https://github.com/AIDC-AI/Ovis" }, { "id": "f28593b7a9d7bd6f521b", "model": "R1-Onevision-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.299, "raw_score": 29.9, "raw_scale": 100, "source_record": "R1-Onevision-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-13", "report_url": "https://arxiv.org/abs/2503.10615" }, { "id": "3253363c22ef6ba17332", "model": "InternVL3-8B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.29, "raw_score": 29.0, "raw_scale": 100, "source_record": "InternVL3-8B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-11", "report_url": "https://arxiv.org/abs/2504.10479" }, { "id": "5475d6513e1af6e83ebc", "model": "VOLD (Qwen2.5-VL-3B, text-only RL)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.28, "raw_score": 28.0, "raw_scale": 100, "source_record": "VOLD (Qwen2.5-VL-3B, text-only RL)|Full test set (3,040 problems); settings in result name", "source_date": "2025-10-27", "report_url": "https://arxiv.org/abs/2510.23497" }, { "id": "4ce3036ef1e6c15826bb", "model": "InternVL3-9B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.276, "raw_score": 27.6, "raw_scale": 100, "source_record": "InternVL3-9B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-11", "report_url": "https://huggingface.co/OpenGVLab/InternVL3-9B" }, { "id": "5689598a738500223038", "model": "Claude3-Opus", "organization": "Anthropic", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.2713, "raw_score": 27.13, "raw_scale": 100, "source_record": "Claude3-Opus|Full test set (3,040 problems); settings in result name", "source_date": "2024-05-04", "report_url": "https://www.anthropic.com/news/claude-3-family/" }, { "id": "e531def7bde9cfb669c7", "model": "MM-Eureka-Qwen-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.26899999999999996, "raw_score": 26.9, "raw_scale": 100, "source_record": "MM-Eureka-Qwen-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-09", "report_url": "https://arxiv.org/abs/2503.07365" }, { "id": "1ada5cae1d7d1b3bac91", "model": "Qwen2.5-VL-DP-7B (MathV-DP)", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.26899999999999996, "raw_score": 26.9, "raw_scale": 100, "source_record": "Qwen2.5-VL-DP-7B (MathV-DP)|Full test set (3,040 problems); settings in result name", "source_date": "2025-07-03", "report_url": "https://arxiv.org/abs/2507.02804" }, { "id": "ffdecc170a8622f9dcf2", "model": "Vision-SR1-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.267, "raw_score": 26.7, "raw_scale": 100, "source_record": "Vision-SR1-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-08-27", "report_url": "https://arxiv.org/abs/2508.19652" }, { "id": "8aa953f90b0af4817333", "model": "MathGLM-Vision-32B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.265, "raw_score": 26.5, "raw_scale": 100, "source_record": "MathGLM-Vision-32B|Full test set (3,040 problems); settings in result name", "source_date": "2024-09-20", "report_url": "https://arxiv.org/abs/2409.13729" }, { "id": "316de02762e0469c9617", "model": "VLAA-Thinker-Qwen2.5VL-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.264, "raw_score": 26.4, "raw_scale": 100, "source_record": "VLAA-Thinker-Qwen2.5VL-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-15", "report_url": "https://arxiv.org/abs/2504.11468" }, { "id": "e4c34e32c55dea26d6b8", "model": "MathCoder-VL-8B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.261, "raw_score": 26.1, "raw_scale": 100, "source_record": "MathCoder-VL-8B|Full test set (3,040 problems); settings in result name", "source_date": "2025-02-16", "report_url": "https://github.com/mathllm/MathCoder/" }, { "id": "e7a263b419db32def384", "model": "OpenVLThinker-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.259, "raw_score": 25.9, "raw_scale": 100, "source_record": "OpenVLThinker-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-21", "report_url": "https://arxiv.org/abs/2503.17352" }, { "id": "03632b19c69bfc00904b", "model": "Ovis2-8B (testmini)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.259, "raw_score": 25.9, "raw_scale": 100, "source_record": "Ovis2-8B (testmini)|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-25", "report_url": "https://github.com/AIDC-AI/Ovis" }, { "id": "5f3bf9d053698138ea45", "model": "Qwen2-VL-72B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.259, "raw_score": 25.9, "raw_scale": 100, "source_record": "Qwen2-VL-72B|Full test set (3,040 problems); settings in result name", "source_date": "2024-08-29", "report_url": "https://qwenlm.github.io/blog/qwen2-vl/" }, { "id": "684e5674d4e56f8ccf45", "model": "LLaVA-OneVision-72B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.253, "raw_score": 25.3, "raw_scale": 100, "source_record": "LLaVA-OneVision-72B|Full test set (3,040 problems); settings in result name", "source_date": "2024-08-06", "report_url": "https://huggingface.co/llava-hf/llava-onevision-qwen2-72b-ov-hf" }, { "id": "86e759615d414121135f", "model": "Qwen2.5-VL-7B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.251, "raw_score": 25.1, "raw_scale": 100, "source_record": "Qwen2.5-VL-7B|Full test set (3,040 problems); settings in result name", "source_date": "2025-01-26" }, { "id": "eabd30f04927a3666600", "model": "TBAC-VLR1-3B-preview", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "TBAC-VLR1-3B-preview|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-21", "report_url": "https://huggingface.co/TencentBAC/TBAC-VLR1-3B-preview" }, { "id": "836d4e6ec80df6a84814", "model": "R1-VL-7B (contamination-flagged)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.247, "raw_score": 24.7, "raw_scale": 100, "source_record": "R1-VL-7B (contamination-flagged)|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-17", "report_url": "https://arxiv.org/abs/2503.12937" }, { "id": "2c8b60b5f1ed4d7c8f72", "model": "X-Reasoner-3B (text-only)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.244, "raw_score": 24.4, "raw_scale": 100, "source_record": "X-Reasoner-3B (text-only)|Full test set (3,040 problems); settings in result name", "source_date": "2025-10-27", "report_url": "https://arxiv.org/abs/2510.23497" }, { "id": "8cae88e76a5941927c52", "model": "CoT GPT4V", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.2398, "raw_score": 23.98, "raw_scale": 100, "source_record": "CoT GPT4V|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://arxiv.org/abs/2310.02255" }, { "id": "6b3c97d54f8a886e8461", "model": "R1-Onevision-3B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "R1-Onevision-3B|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-13", "report_url": "https://arxiv.org/abs/2503.10615" }, { "id": "eb60d8e02c8b72a60c21", "model": "MiniCPM-V 2.6", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.23399999999999999, "raw_score": 23.4, "raw_scale": 100, "source_record": "MiniCPM-V 2.6|Full test set (3,040 problems); settings in result name", "source_date": "2024-08-06", "report_url": "https://huggingface.co/openbmb/MiniCPM-V-2_6" }, { "id": "c087ccbce71c190a2f9b", "model": "InternVL2.5-26B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.231, "raw_score": 23.1, "raw_scale": 100, "source_record": "InternVL2.5-26B|Full test set (3,040 problems); settings in result name", "source_date": "2024-12-05", "report_url": "https://huggingface.co/OpenGVLab/InternVL2_5-26B" }, { "id": "71b99aaf999ec60827e7", "model": "GPT4V", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.22760000000000002, "raw_score": 22.76, "raw_scale": 100, "source_record": "GPT4V|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://arxiv.org/abs/2310.02255" }, { "id": "1f7121ea6d5dcd65e4ec", "model": "InternVL3-2B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.217, "raw_score": 21.7, "raw_scale": 100, "source_record": "InternVL3-2B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-11", "report_url": "https://huggingface.co/OpenGVLab/InternVL3-2B" }, { "id": "53c46d90b6528a5bf53e", "model": "MathCoder-VL-2B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.217, "raw_score": 21.7, "raw_scale": 100, "source_record": "MathCoder-VL-2B|Full test set (3,040 problems); settings in result name", "source_date": "2025-02-16", "report_url": "https://github.com/mathllm/MathCoder/" }, { "id": "caf484cf15782b04c995", "model": "MiniCPM-o 2.6", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.217, "raw_score": 21.7, "raw_scale": 100, "source_record": "MiniCPM-o 2.6|Full test set (3,040 problems); settings in result name", "source_date": "2025-01-13", "report_url": "https://huggingface.co/openbmb/MiniCPM-o-2_6" }, { "id": "e8344f3e67fb12a513fe", "model": "Kimi-VL (base, non-thinking)", "organization": "Moonshot AI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.214, "raw_score": 21.4, "raw_scale": 100, "source_record": "Kimi-VL (base, non-thinking)|Full test set (3,040 problems); settings in result name", "source_date": "2025-06-23", "report_url": "https://arxiv.org/pdf/2504.07491" }, { "id": "decec8ebe21eb1ea5b42", "model": "Qwen2.5-VL-3B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.212, "raw_score": 21.2, "raw_scale": 100, "source_record": "Qwen2.5-VL-3B|Full test set (3,040 problems); settings in result name", "source_date": "2025-01-26" }, { "id": "5e8c03b2b4fb71afc7c8", "model": "Multimath-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.207, "raw_score": 20.7, "raw_scale": 100, "source_record": "Multimath-7B|Full test set (3,040 problems); settings in result name", "source_date": "2024-08-30", "report_url": "https://arxiv.org/abs/2409.00147" }, { "id": "0183271e2ca79f8056cb", "model": "InternVL2.5-8B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.19699999999999998, "raw_score": 19.7, "raw_scale": 100, "source_record": "InternVL2.5-8B|Full test set (3,040 problems); settings in result name", "source_date": "2024-12-05", "report_url": "https://huggingface.co/OpenGVLab/InternVL2_5-8B" }, { "id": "ec45cde17c042ae002fd", "model": "Mulberry-Qwen2VL-7B (contamination-flagged)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.195, "raw_score": 19.5, "raw_scale": 100, "source_record": "Mulberry-Qwen2VL-7B (contamination-flagged)|Full test set (3,040 problems); settings in result name", "source_date": "2024-12-24", "report_url": "https://arxiv.org/abs/2412.18319" }, { "id": "a1f4d4727b1cfa50bc96", "model": "Gemini-1.5 Pro", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.1924, "raw_score": 19.24, "raw_scale": 100, "source_record": "Gemini-1.5 Pro|Full test set (3,040 problems); settings in result name", "source_date": "2024-05-17", "report_url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf" }, { "id": "db0e50b34c18f29d36ca", "model": "InternVL3-1B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.188, "raw_score": 18.8, "raw_scale": 100, "source_record": "InternVL3-1B|Full test set (3,040 problems); settings in result name", "source_date": "2025-04-11", "report_url": "https://huggingface.co/OpenGVLab/InternVL3-1B" }, { "id": "19886aa2fd7b60fae008", "model": "Ovis1.6-Gemma2-9B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.18780000000000002, "raw_score": 18.78, "raw_scale": 100, "source_record": "Ovis1.6-Gemma2-9B|Full test set (3,040 problems); settings in result name", "source_date": "2024-09-19", "report_url": "https://github.com/AIDC-AI/Ovis" }, { "id": "4438e0f347aaa3150edd", "model": "MAVIS-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.18600000000000003, "raw_score": 18.6, "raw_scale": 100, "source_record": "MAVIS-7B|Full test set (3,040 problems); settings in result name", "source_date": "2024-07-11", "report_url": "https://arxiv.org/abs/2407.08739" }, { "id": "bbddcd4f5aef1cd9bf92", "model": "Aquila-VL-2B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.179, "raw_score": 17.9, "raw_scale": 100, "source_record": "Aquila-VL-2B|Full test set (3,040 problems); settings in result name", "source_date": "2024-10-25", "report_url": "https://huggingface.co/BAAI/Aquila-VL-2B-llava-qwen" }, { "id": "3c4860aaaf0551ebf8cc", "model": "Ovis2-2B (testmini)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.177, "raw_score": 17.7, "raw_scale": 100, "source_record": "Ovis2-2B (testmini)|Full test set (3,040 problems); settings in result name", "source_date": "2025-03-25", "report_url": "https://github.com/AIDC-AI/Ovis" }, { "id": "8757b6689ffe1ac31c0d", "model": "Qwen2-VL-DP-7B (MathV-DP)", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.177, "raw_score": 17.7, "raw_scale": 100, "source_record": "Qwen2-VL-DP-7B (MathV-DP)|Full test set (3,040 problems); settings in result name", "source_date": "2025-07-03", "report_url": "https://arxiv.org/abs/2507.02804" }, { "id": "944c988a84e34540d073", "model": "Gemini Pro", "organization": "Google", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.1766, "raw_score": 17.66, "raw_scale": 100, "source_record": "Gemini Pro|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://arxiv.org/abs/2312.11805" }, { "id": "d47684085f16d537f13d", "model": "InternVL-Chat-V1-2-Plus", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.1697, "raw_score": 16.97, "raw_scale": 100, "source_record": "InternVL-Chat-V1-2-Plus|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-22", "report_url": "https://github.com/OpenGVLab/InternVL" }, { "id": "cbcecabf48909f57b37d", "model": "Qwen2-VL-7B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.163, "raw_score": 16.3, "raw_scale": 100, "source_record": "Qwen2-VL-7B|Full test set (3,040 problems); settings in result name", "source_date": "2024-08-29", "report_url": "https://qwenlm.github.io/blog/qwen2-vl/" }, { "id": "a2713975150e8c0e4f2c", "model": "Math-LLaVA-13B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.15689999999999998, "raw_score": 15.69, "raw_scale": 100, "source_record": "Math-LLaVA-13B|Full test set (3,040 problems); settings in result name", "source_date": "2024-06-26", "report_url": "https://github.com/HZQ950419/Math-LLaVA" }, { "id": "c362afecd8fb2205cfea", "model": "Qwen-VL-Max", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.1559, "raw_score": 15.59, "raw_scale": 100, "source_record": "Qwen-VL-Max|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://github.com/QwenLM/Qwen-VL" }, { "id": "fc79e01312bdc2e9c542", "model": "InternVL2.5-2B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.147, "raw_score": 14.7, "raw_scale": 100, "source_record": "InternVL2.5-2B|Full test set (3,040 problems); settings in result name", "source_date": "2024-12-05", "report_url": "https://huggingface.co/OpenGVLab/InternVL2_5-2B" }, { "id": "7871aa60588e67815fca", "model": "InternLM-XComposer2-VL", "organization": "Shanghai AI Laboratory", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.1454, "raw_score": 14.54, "raw_scale": 100, "source_record": "InternLM-XComposer2-VL|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://github.com/InternLM/InternLM-XComposer" }, { "id": "f5d8ea98d61c31e4bb82", "model": "InternVL2.5-1B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.14400000000000002, "raw_score": 14.4, "raw_scale": 100, "source_record": "InternVL2.5-1B|Full test set (3,040 problems); settings in result name", "source_date": "2024-12-05", "report_url": "https://huggingface.co/OpenGVLab/InternVL2_5-1B" }, { "id": "7a2c1d48ce976ac30146", "model": "GPT 4-CoT (caption)", "organization": "OpenAI", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.131, "raw_score": 13.1, "raw_scale": 100, "source_record": "GPT 4-CoT (caption)|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://arxiv.org/abs/2310.02255" }, { "id": "aaeafc809a934d9f07d2", "model": "Qwen2-VL-2B", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.124, "raw_score": 12.4, "raw_scale": 100, "source_record": "Qwen2-VL-2B|Full test set (3,040 problems); settings in result name", "source_date": "2024-08-29", "report_url": "https://qwenlm.github.io/blog/qwen2-vl/" }, { "id": "2494387579394ff1e18c", "model": "ShareGPT4V-13B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.1188, "raw_score": 11.88, "raw_scale": 100, "source_record": "ShareGPT4V-13B|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://sharegpt4v.github.io/" }, { "id": "ac52150ca28feab05362", "model": "LLaVA-v1.5-13B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.1112, "raw_score": 11.12, "raw_scale": 100, "source_record": "LLaVA-v1.5-13B|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://llava-vl.github.io/" }, { "id": "11e1ea87b09eea91a09f", "model": "Qwen-VL-Plus", "organization": "Alibaba", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.1072, "raw_score": 10.72, "raw_scale": 100, "source_record": "Qwen-VL-Plus|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://github.com/QwenLM/Qwen-VL" }, { "id": "e9f3f677788df5f9340d", "model": "ShareGPT4V-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.10529999999999999, "raw_score": 10.53, "raw_scale": 100, "source_record": "ShareGPT4V-7B|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://sharegpt4v.github.io/" }, { "id": "930c0f7ab43994a872f1", "model": "SPHINX (V2)", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.09699999999999999, "raw_score": 9.7, "raw_scale": 100, "source_record": "SPHINX (V2)|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://github.com/Alpha-VLLM/LLaMA2-Accessory/tree/main/SPHINX" }, { "id": "64b583f7c4f1073dc6b0", "model": "LLaVA-v1.5-7B", "organization": "", "configuration": "Full test set (3,040 problems); settings in result name", "score": 0.0852, "raw_score": 8.52, "raw_scale": 100, "source_record": "LLaVA-v1.5-7B|Full test set (3,040 problems); settings in result name", "source_date": "2024-02-21", "report_url": "https://llava-vl.github.io/" } ] }, { "id": "charxiv-reasoning", "name": "CharXiv (reasoning)", "group": "视觉与视频", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Validation split; reasoning accuracy. Descriptive and reasoning questions are separate cohorts; human/random baselines are excluded.", "zh": "验证集的推理准确率；两类问题分开统计，不含人类和随机基线。" }, "source": { "data_url": "https://charxiv.github.io/data/val_result.csv", "sha256": "11fe8260714bffef6062091524d3f9103e62d704522bea37408de4b7f34607c0", "retrieved_at": "2026-11-01T16:40:34.531649+00:00", "name": "CharXiv", "url": "https://charxiv.github.io/", "selector": "val_result.csv: Overall column 3 (zero-indexed)", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "b9f43901ea801ccc741d", "model": "o3 (high)", "organization": "", "configuration": "Validation; reasoning", "score": 0.7859999999999999, "raw_score": 78.6, "raw_scale": 100, "source_record": "o3 (high)|Validation; reasoning" }, { "id": "89738b7dec774fab0fc1", "model": "o4 mini (high)", "organization": "", "configuration": "Validation; reasoning", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "o4 mini (high)|Validation; reasoning" }, { "id": "06d4ce74ac7b55b66989", "model": "Claude 3.7 Sonnet", "organization": "Anthropic", "configuration": "Validation; reasoning", "score": 0.642, "raw_score": 64.2, "raw_scale": 100, "source_record": "Claude 3.7 Sonnet|Validation; reasoning" }, { "id": "c8ae8d0f0d9f1b5f3c64", "model": "Claude 3.5 Sonnet", "organization": "Anthropic", "configuration": "Validation; reasoning", "score": 0.602, "raw_score": 60.2, "raw_scale": 100, "source_record": "Claude 3.5 Sonnet|Validation; reasoning" }, { "id": "2b48ad35d9daad53e660", "model": "GPT 4.1 mini", "organization": "OpenAI", "configuration": "Validation; reasoning", "score": 0.568, "raw_score": 56.8, "raw_scale": 100, "source_record": "GPT 4.1 mini|Validation; reasoning" }, { "id": "7379442e9ca0ce3a2b18", "model": "GPT 4.1", "organization": "OpenAI", "configuration": "Validation; reasoning", "score": 0.5670000000000001, "raw_score": 56.7, "raw_scale": 100, "source_record": "GPT 4.1|Validation; reasoning" }, { "id": "7f2422f5cdbb8b2d5475", "model": "GPT 4.5", "organization": "OpenAI", "configuration": "Validation; reasoning", "score": 0.5539999999999999, "raw_score": 55.4, "raw_scale": 100, "source_record": "GPT 4.5|Validation; reasoning" }, { "id": "40bbd22c89703b189b70", "model": "o1 (high)", "organization": "", "configuration": "Validation; reasoning", "score": 0.551, "raw_score": 55.1, "raw_scale": 100, "source_record": "o1 (high)|Validation; reasoning" }, { "id": "6c11a334c5ad12c53eb3", "model": "Doubao 1.5 Pro", "organization": "", "configuration": "Validation; reasoning", "score": 0.544, "raw_score": 54.4, "raw_scale": 100, "source_record": "Doubao 1.5 Pro|Validation; reasoning" }, { "id": "3bc1139f73b10ba239af", "model": "o1", "organization": "OpenAI", "configuration": "Validation; reasoning", "score": 0.526, "raw_score": 52.6, "raw_scale": 100, "source_record": "o1|Validation; reasoning" }, { "id": "b63b82cc71634690d23c", "model": "GPT-4o 241120", "organization": "OpenAI", "configuration": "Validation; reasoning", "score": 0.505, "raw_score": 50.5, "raw_scale": 100, "source_record": "GPT-4o 241120|Validation; reasoning" }, { "id": "edceb9c37beb866f6d1e", "model": "Qwen2.5-VL 72B", "organization": "Alibaba", "configuration": "Validation; reasoning", "score": 0.49700000000000005, "raw_score": 49.7, "raw_scale": 100, "source_record": "Qwen2.5-VL 72B|Validation; reasoning" }, { "id": "7f9712349cd8e8da4de7", "model": "GPT-4o 240513", "organization": "OpenAI", "configuration": "Validation; reasoning", "score": 0.47100000000000003, "raw_score": 47.1, "raw_scale": 100, "source_record": "GPT-4o 240513|Validation; reasoning" }, { "id": "6852b481ae3e037b7475", "model": "InternVL3 38B", "organization": "", "configuration": "Validation; reasoning", "score": 0.46399999999999997, "raw_score": 46.4, "raw_scale": 100, "source_record": "InternVL3 38B|Validation; reasoning" }, { "id": "aa31b9c680a0be99dc46", "model": "InternVL3 78B", "organization": "", "configuration": "Validation; reasoning", "score": 0.46, "raw_score": 46.0, "raw_scale": 100, "source_record": "InternVL3 78B|Validation; reasoning" }, { "id": "11f7a3e1b3bfa557561b", "model": "Gemini 1.5 Pro", "organization": "Google", "configuration": "Validation; reasoning", "score": 0.433, "raw_score": 43.3, "raw_scale": 100, "source_record": "Gemini 1.5 Pro|Validation; reasoning" }, { "id": "585e00cd7aba40943065", "model": "InternVL3 14B", "organization": "", "configuration": "Validation; reasoning", "score": 0.431, "raw_score": 43.1, "raw_scale": 100, "source_record": "InternVL3 14B|Validation; reasoning" }, { "id": "47d5e03235c5ff30d3d6", "model": "Qwen2-VL 72B", "organization": "Alibaba", "configuration": "Validation; reasoning", "score": 0.43, "raw_score": 43.0, "raw_scale": 100, "source_record": "Qwen2-VL 72B|Validation; reasoning" }, { "id": "ed263ab42d4cfb1570b2", "model": "Qwen2.5-VL 7B", "organization": "Alibaba", "configuration": "Validation; reasoning", "score": 0.425, "raw_score": 42.5, "raw_scale": 100, "source_record": "Qwen2.5-VL 7B|Validation; reasoning" }, { "id": "d336f313efc6ec90b649", "model": "InternVL V2.5 38B", "organization": "", "configuration": "Validation; reasoning", "score": 0.424, "raw_score": 42.4, "raw_scale": 100, "source_record": "InternVL V2.5 38B|Validation; reasoning" }, { "id": "1ba0c9882a983641dbb2", "model": "InternVL V2.5 78B", "organization": "", "configuration": "Validation; reasoning", "score": 0.424, "raw_score": 42.4, "raw_scale": 100, "source_record": "InternVL V2.5 78B|Validation; reasoning" }, { "id": "282b0cd07cdd8373c11f", "model": "Pixtral 12B", "organization": "", "configuration": "Validation; reasoning", "score": 0.424, "raw_score": 42.4, "raw_scale": 100, "source_record": "Pixtral 12B|Validation; reasoning" }, { "id": "c3359abbc920b1b33d47", "model": "GPT 4.1 nano", "organization": "OpenAI", "configuration": "Validation; reasoning", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "GPT 4.1 nano|Validation; reasoning" }, { "id": "19964dd18cfc9a037398", "model": "InternVL Chat V2.0 Pro", "organization": "", "configuration": "Validation; reasoning", "score": 0.39799999999999996, "raw_score": 39.8, "raw_scale": 100, "source_record": "InternVL Chat V2.0 Pro|Validation; reasoning" }, { "id": "7a51ac6c4a3855d35576", "model": "InternVL Chat V2.0 76B", "organization": "", "configuration": "Validation; reasoning", "score": 0.389, "raw_score": 38.9, "raw_scale": 100, "source_record": "InternVL Chat V2.0 76B|Validation; reasoning" }, { "id": "e9d3da5a0ddb63e9d7fb", "model": "InternVL3 9B", "organization": "", "configuration": "Validation; reasoning", "score": 0.38, "raw_score": 38.0, "raw_scale": 100, "source_record": "InternVL3 9B|Validation; reasoning" }, { "id": "c62c51188c6a843dbca2", "model": "InternVL3 8B", "organization": "", "configuration": "Validation; reasoning", "score": 0.376, "raw_score": 37.6, "raw_scale": 100, "source_record": "InternVL3 8B|Validation; reasoning" }, { "id": "11769ff42e707ada962c", "model": "Llama 3.2 Vision 90B", "organization": "Meta", "configuration": "Validation; reasoning", "score": 0.375, "raw_score": 37.5, "raw_scale": 100, "source_record": "Llama 3.2 Vision 90B|Validation; reasoning" }, { "id": "b69cef3c0c92aa7d6df6", "model": "GPT-4V", "organization": "OpenAI", "configuration": "Validation; reasoning", "score": 0.371, "raw_score": 37.1, "raw_scale": 100, "source_record": "GPT-4V|Validation; reasoning" }, { "id": "4cff06b02bea2a61e586", "model": "NVLM-D 72B", "organization": "", "configuration": "Validation; reasoning", "score": 0.368, "raw_score": 36.8, "raw_scale": 100, "source_record": "NVLM-D 72B|Validation; reasoning" }, { "id": "1980647a2a9c8e7d5b88", "model": "Molmo 72B", "organization": "", "configuration": "Validation; reasoning", "score": 0.365, "raw_score": 36.5, "raw_scale": 100, "source_record": "Molmo 72B|Validation; reasoning" }, { "id": "430d159cd43eeb73306a", "model": "Llava OneVision 72B", "organization": "", "configuration": "Validation; reasoning", "score": 0.364, "raw_score": 36.4, "raw_scale": 100, "source_record": "Llava OneVision 72B|Validation; reasoning" }, { "id": "59fbc78754fc8ebbdf31", "model": "InternVL V2.5 26B", "organization": "", "configuration": "Validation; reasoning", "score": 0.359, "raw_score": 35.9, "raw_scale": 100, "source_record": "InternVL V2.5 26B|Validation; reasoning" }, { "id": "fd3cfac40cbc29576505", "model": "Qwen2-VL 7B", "organization": "Alibaba", "configuration": "Validation; reasoning", "score": 0.34600000000000003, "raw_score": 34.6, "raw_scale": 100, "source_record": "Qwen2-VL 7B|Validation; reasoning" }, { "id": "acdfacc0d6cba06ae201", "model": "GPT-4o Mini", "organization": "OpenAI", "configuration": "Validation; reasoning", "score": 0.341, "raw_score": 34.1, "raw_scale": 100, "source_record": "GPT-4o Mini|Validation; reasoning" }, { "id": "384c54535fe0f19c3a9b", "model": "Gemini 1.5 Flash", "organization": "Google", "configuration": "Validation; reasoning", "score": 0.33899999999999997, "raw_score": 33.9, "raw_scale": 100, "source_record": "Gemini 1.5 Flash|Validation; reasoning" }, { "id": "971aba70cfb3a3ecd2ab", "model": "InternVL Chat V2.0 26B", "organization": "", "configuration": "Validation; reasoning", "score": 0.33399999999999996, "raw_score": 33.4, "raw_scale": 100, "source_record": "InternVL Chat V2.0 26B|Validation; reasoning" }, { "id": "098c0eaab03965684d31", "model": "InternVL V2.5 8B", "organization": "", "configuration": "Validation; reasoning", "score": 0.32899999999999996, "raw_score": 32.9, "raw_scale": 100, "source_record": "InternVL V2.5 8B|Validation; reasoning" }, { "id": "fe5645265c53b9e0d6f6", "model": "Phi-3.5 Vision", "organization": "Microsoft", "configuration": "Validation; reasoning", "score": 0.327, "raw_score": 32.7, "raw_scale": 100, "source_record": "Phi-3.5 Vision|Validation; reasoning" }, { "id": "42d09a5b720e1b7fee40", "model": "Claude 3 Sonnet", "organization": "Anthropic", "configuration": "Validation; reasoning", "score": 0.322, "raw_score": 32.2, "raw_scale": 100, "source_record": "Claude 3 Sonnet|Validation; reasoning" }, { "id": "86b37673acdef6af2bb0", "model": "Claude 3 Haiku", "organization": "Anthropic", "configuration": "Validation; reasoning", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "Claude 3 Haiku|Validation; reasoning" }, { "id": "90dd572c0c70cb72449f", "model": "Phi-3 Vision", "organization": "Microsoft", "configuration": "Validation; reasoning", "score": 0.316, "raw_score": 31.6, "raw_scale": 100, "source_record": "Phi-3 Vision|Validation; reasoning" }, { "id": "80297d1ce3dc69f158de", "model": "Qwen2.5-VL 3B", "organization": "Alibaba", "configuration": "Validation; reasoning", "score": 0.313, "raw_score": 31.3, "raw_scale": 100, "source_record": "Qwen2.5-VL 3B|Validation; reasoning" }, { "id": "ec4af251bdcd007cc167", "model": "Llama 3.2 Vision 11B", "organization": "Meta", "configuration": "Validation; reasoning", "score": 0.312, "raw_score": 31.2, "raw_scale": 100, "source_record": "Llama 3.2 Vision 11B|Validation; reasoning" }, { "id": "68f526e009e4495eade8", "model": "MiniCPM-V2.6 (Upsize+CoT)", "organization": "", "configuration": "Validation; reasoning", "score": 0.31, "raw_score": 31.0, "raw_scale": 100, "source_record": "MiniCPM-V2.6 (Upsize+CoT)|Validation; reasoning" }, { "id": "e2bc6fc89c5912152fa3", "model": "Claude 3 Opus", "organization": "Anthropic", "configuration": "Validation; reasoning", "score": 0.302, "raw_score": 30.2, "raw_scale": 100, "source_record": "Claude 3 Opus|Validation; reasoning" }, { "id": "effa584e5a54eb4256d1", "model": "InternVL Chat V1.5", "organization": "", "configuration": "Validation; reasoning", "score": 0.292, "raw_score": 29.2, "raw_scale": 100, "source_record": "InternVL Chat V1.5|Validation; reasoning" }, { "id": "3866b7834ad906e28167", "model": "GLM 4V 9B", "organization": "Z AI", "configuration": "Validation; reasoning", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "GLM 4V 9B|Validation; reasoning" }, { "id": "a57742a00ab248c0ed7d", "model": "Reka Core", "organization": "", "configuration": "Validation; reasoning", "score": 0.289, "raw_score": 28.9, "raw_scale": 100, "source_record": "Reka Core|Validation; reasoning" }, { "id": "dc0a3c3a7aa47f5a5947", "model": "Ovis 1.5 Gemma2 9B", "organization": "", "configuration": "Validation; reasoning", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "Ovis 1.5 Gemma2 9B|Validation; reasoning" }, { "id": "87f462bf48c06fda223c", "model": "InternVL3 2B", "organization": "", "configuration": "Validation; reasoning", "score": 0.28300000000000003, "raw_score": 28.3, "raw_scale": 100, "source_record": "InternVL3 2B|Validation; reasoning" }, { "id": "14973444559237cc14cc", "model": "Ovis 1.5 Llama3 8B", "organization": "", "configuration": "Validation; reasoning", "score": 0.282, "raw_score": 28.2, "raw_scale": 100, "source_record": "Ovis 1.5 Llama3 8B|Validation; reasoning" }, { "id": "057808444cd07146242a", "model": "Cambrian 34B", "organization": "", "configuration": "Validation; reasoning", "score": 0.273, "raw_score": 27.3, "raw_scale": 100, "source_record": "Cambrian 34B|Validation; reasoning" }, { "id": "4f6f9494b14402d1b9e2", "model": "MiniCPM-V2.6 (Upsize)", "organization": "", "configuration": "Validation; reasoning", "score": 0.271, "raw_score": 27.1, "raw_scale": 100, "source_record": "MiniCPM-V2.6 (Upsize)|Validation; reasoning" }, { "id": "16d3c351e55176875805", "model": "Reka Flash", "organization": "", "configuration": "Validation; reasoning", "score": 0.266, "raw_score": 26.6, "raw_scale": 100, "source_record": "Reka Flash|Validation; reasoning" }, { "id": "4fa96b08cb520ecb3171", "model": "Molmo 7B D", "organization": "", "configuration": "Validation; reasoning", "score": 0.264, "raw_score": 26.4, "raw_scale": 100, "source_record": "Molmo 7B D|Validation; reasoning" }, { "id": "9a8d68d07e3127cfa365", "model": "InternLM XComposer2 4KHD", "organization": "Shanghai AI Laboratory", "configuration": "Validation; reasoning", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "InternLM XComposer2 4KHD|Validation; reasoning" }, { "id": "95e21066c6c4a86d7db7", "model": "Mini Gemini HD Yi 34B", "organization": "", "configuration": "Validation; reasoning", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "Mini Gemini HD Yi 34B|Validation; reasoning" }, { "id": "fff7738edff3593582bd", "model": "InternVL V2.5 4B", "organization": "", "configuration": "Validation; reasoning", "score": 0.249, "raw_score": 24.9, "raw_scale": 100, "source_record": "InternVL V2.5 4B|Validation; reasoning" }, { "id": "262108475506829c46b5", "model": "MiniCPM-V2.5", "organization": "", "configuration": "Validation; reasoning", "score": 0.249, "raw_score": 24.9, "raw_scale": 100, "source_record": "MiniCPM-V2.5|Validation; reasoning" }, { "id": "08bcf8e3eefe97008cf4", "model": "Qwen VL Max", "organization": "Alibaba", "configuration": "Validation; reasoning", "score": 0.247, "raw_score": 24.7, "raw_scale": 100, "source_record": "Qwen VL Max|Validation; reasoning" }, { "id": "3820cfbf498aa20a68d7", "model": "Qwen2-VL 2B", "organization": "Alibaba", "configuration": "Validation; reasoning", "score": 0.245, "raw_score": 24.5, "raw_scale": 100, "source_record": "Qwen2-VL 2B|Validation; reasoning" }, { "id": "79cdd1257c43750fba8a", "model": "Molmo 7B O", "organization": "", "configuration": "Validation; reasoning", "score": 0.243, "raw_score": 24.3, "raw_scale": 100, "source_record": "Molmo 7B O|Validation; reasoning" }, { "id": "891260d8af4cfa5df485", "model": "VILA 1.5 40B", "organization": "", "configuration": "Validation; reasoning", "score": 0.24, "raw_score": 24.0, "raw_scale": 100, "source_record": "VILA 1.5 40B|Validation; reasoning" }, { "id": "3dd5e4aa7561403380d5", "model": "Llava OneVision 7B", "organization": "", "configuration": "Validation; reasoning", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "Llava OneVision 7B|Validation; reasoning" }, { "id": "8f1e17a43b05b1bb4983", "model": "Reka Edge", "organization": "", "configuration": "Validation; reasoning", "score": 0.235, "raw_score": 23.5, "raw_scale": 100, "source_record": "Reka Edge|Validation; reasoning" }, { "id": "d6011149d93e66e4ff0f", "model": "Gemini 1.0 Pro", "organization": "Google", "configuration": "Validation; reasoning", "score": 0.228, "raw_score": 22.8, "raw_scale": 100, "source_record": "Gemini 1.0 Pro|Validation; reasoning" }, { "id": "622db69aacd49dc78bd8", "model": "LLaVA 1.6 Yi 34B", "organization": "", "configuration": "Validation; reasoning", "score": 0.225, "raw_score": 22.5, "raw_scale": 100, "source_record": "LLaVA 1.6 Yi 34B|Validation; reasoning" }, { "id": "7947dd5291383565337d", "model": "InternVL V2.5 2B", "organization": "", "configuration": "Validation; reasoning", "score": 0.213, "raw_score": 21.3, "raw_scale": 100, "source_record": "InternVL V2.5 2B|Validation; reasoning" }, { "id": "5929aef6f7c299667d30", "model": "InternVL3 1B", "organization": "", "configuration": "Validation; reasoning", "score": 0.21, "raw_score": 21.0, "raw_scale": 100, "source_record": "InternVL3 1B|Validation; reasoning" }, { "id": "39ee8e211a950e7e297f", "model": "InternVL V2.5 1B", "organization": "", "configuration": "Validation; reasoning", "score": 0.19, "raw_score": 19.0, "raw_scale": 100, "source_record": "InternVL V2.5 1B|Validation; reasoning" }, { "id": "bc9c6942968d8d913763", "model": "Mini Gemini HD Llama3 8B", "organization": "", "configuration": "Validation; reasoning", "score": 0.19, "raw_score": 19.0, "raw_scale": 100, "source_record": "Mini Gemini HD Llama3 8B|Validation; reasoning" }, { "id": "4b1782ea4e82a3fc1047", "model": "CogAgent", "organization": "", "configuration": "Validation; reasoning", "score": 0.188, "raw_score": 18.8, "raw_scale": 100, "source_record": "CogAgent|Validation; reasoning" }, { "id": "be54ec68e1ad554a5a02", "model": "InternLM XComposer2", "organization": "Shanghai AI Laboratory", "configuration": "Validation; reasoning", "score": 0.187, "raw_score": 18.7, "raw_scale": 100, "source_record": "InternLM XComposer2|Validation; reasoning" }, { "id": "5528cb93ec82026c7e0a", "model": "MiniCPM-V2", "organization": "", "configuration": "Validation; reasoning", "score": 0.185, "raw_score": 18.5, "raw_scale": 100, "source_record": "MiniCPM-V2|Validation; reasoning" }, { "id": "48e53789331cc0def925", "model": "IDEFICS 2", "organization": "", "configuration": "Validation; reasoning", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "IDEFICS 2|Validation; reasoning" }, { "id": "82bfb10c5d0f5d38a92b", "model": "MolmoE 1B", "organization": "", "configuration": "Validation; reasoning", "score": 0.18, "raw_score": 18.0, "raw_scale": 100, "source_record": "MolmoE 1B|Validation; reasoning" }, { "id": "df2790be8232f168bf50", "model": "IDEFICS 2 Chatty", "organization": "", "configuration": "Validation; reasoning", "score": 0.17800000000000002, "raw_score": 17.8, "raw_scale": 100, "source_record": "IDEFICS 2 Chatty|Validation; reasoning" }, { "id": "b0ce31397534068f789a", "model": "MoAI", "organization": "", "configuration": "Validation; reasoning", "score": 0.175, "raw_score": 17.5, "raw_scale": 100, "source_record": "MoAI|Validation; reasoning" }, { "id": "f209090f645ec72a303f", "model": "DeepSeek VL", "organization": "DeepSeek", "configuration": "Validation; reasoning", "score": 0.171, "raw_score": 17.1, "raw_scale": 100, "source_record": "DeepSeek VL|Validation; reasoning" }, { "id": "1bf7aae552e298dab7b4", "model": "DocOwl 1.5 Chat", "organization": "", "configuration": "Validation; reasoning", "score": 0.17, "raw_score": 17.0, "raw_scale": 100, "source_record": "DocOwl 1.5 Chat|Validation; reasoning" }, { "id": "a98ee613e92aa3383da2", "model": "SPHINX V2", "organization": "", "configuration": "Validation; reasoning", "score": 0.161, "raw_score": 16.1, "raw_scale": 100, "source_record": "SPHINX V2|Validation; reasoning" }, { "id": "50fdd7a2fbd38b28c593", "model": "Qwen VL Plus", "organization": "Alibaba", "configuration": "Validation; reasoning", "score": 0.16, "raw_score": 16.0, "raw_scale": 100, "source_record": "Qwen VL Plus|Validation; reasoning" }, { "id": "9b0242e6888b8a9ad0c3", "model": "Llava OneVision 0.5B", "organization": "", "configuration": "Validation; reasoning", "score": 0.156, "raw_score": 15.6, "raw_scale": 100, "source_record": "Llava OneVision 0.5B|Validation; reasoning" }, { "id": "c4a9cf7da284e071f7af", "model": "UReader", "organization": "", "configuration": "Validation; reasoning", "score": 0.14300000000000002, "raw_score": 14.3, "raw_scale": 100, "source_record": "UReader|Validation; reasoning" }, { "id": "f669991aeb45f1289e33", "model": "ChartLlama", "organization": "", "configuration": "Validation; reasoning", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "ChartLlama|Validation; reasoning" }, { "id": "a834d533cb6f2451b9c5", "model": "LLaVA 1.6 Mistral 7B", "organization": "", "configuration": "Validation; reasoning", "score": 0.139, "raw_score": 13.9, "raw_scale": 100, "source_record": "LLaVA 1.6 Mistral 7B|Validation; reasoning" }, { "id": "41f479c6f5b11706d994", "model": "ChartGemma", "organization": "", "configuration": "Validation; reasoning", "score": 0.125, "raw_score": 12.5, "raw_scale": 100, "source_record": "ChartGemma|Validation; reasoning" }, { "id": "92aad5274d4a87bfa453", "model": "ChartAssistant", "organization": "", "configuration": "Validation; reasoning", "score": 0.11699999999999999, "raw_score": 11.7, "raw_scale": 100, "source_record": "ChartAssistant|Validation; reasoning" }, { "id": "e2a66bd6ec6fa62ee185", "model": "ChartInstruct-FlanT5", "organization": "", "configuration": "Validation; reasoning", "score": 0.11699999999999999, "raw_score": 11.7, "raw_scale": 100, "source_record": "ChartInstruct-FlanT5|Validation; reasoning" }, { "id": "a568010130370237c29a", "model": "DocOwl 1.5 Omni", "organization": "", "configuration": "Validation; reasoning", "score": 0.091, "raw_score": 9.1, "raw_scale": 100, "source_record": "DocOwl 1.5 Omni|Validation; reasoning" }, { "id": "b6148939db482766fa97", "model": "ChartInstruct-Llama2", "organization": "", "configuration": "Validation; reasoning", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "ChartInstruct-Llama2|Validation; reasoning" }, { "id": "39dfedaea7d42744bb92", "model": "TinyChart", "organization": "", "configuration": "Validation; reasoning", "score": 0.083, "raw_score": 8.3, "raw_scale": 100, "source_record": "TinyChart|Validation; reasoning" }, { "id": "0dee96a86728af3f3a40", "model": "UniChart-ChartQA", "organization": "", "configuration": "Validation; reasoning", "score": 0.057, "raw_score": 5.7, "raw_scale": 100, "source_record": "UniChart-ChartQA|Validation; reasoning" }, { "id": "169e2b52b873601d2137", "model": "TextMonkey", "organization": "", "configuration": "Validation; reasoning", "score": 0.039, "raw_score": 3.9, "raw_scale": 100, "source_record": "TextMonkey|Validation; reasoning" } ] }, { "id": "charxiv-descriptive", "name": "CharXiv (descriptive)", "group": "视觉与视频", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Validation split; descriptive accuracy. Descriptive and reasoning questions are separate cohorts; human/random baselines are excluded.", "zh": "验证集的描述准确率；两类问题分开统计，不含人类和随机基线。" }, "source": { "data_url": "https://charxiv.github.io/data/val_result.csv", "sha256": "11fe8260714bffef6062091524d3f9103e62d704522bea37408de4b7f34607c0", "retrieved_at": "2026-11-01T16:40:34.531649+00:00", "name": "CharXiv", "url": "https://charxiv.github.io/", "selector": "val_result.csv: Overall column 8 (zero-indexed)", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "88dae3b012c284703683", "model": "o3 (high)", "organization": "", "configuration": "Validation; descriptive", "score": 0.95, "raw_score": 95.0, "raw_scale": 100, "source_record": "o3 (high)|Validation; descriptive" }, { "id": "18b66ff5906a595ac93e", "model": "o4 mini (high)", "organization": "", "configuration": "Validation; descriptive", "score": 0.943, "raw_score": 94.3, "raw_scale": 100, "source_record": "o4 mini (high)|Validation; descriptive" }, { "id": "0f34ec50ef529ecbf4d5", "model": "GPT 4.5", "organization": "OpenAI", "configuration": "Validation; descriptive", "score": 0.9, "raw_score": 90.0, "raw_scale": 100, "source_record": "GPT 4.5|Validation; descriptive" }, { "id": "ffc4932edbfee9630627", "model": "o1 (high)", "organization": "", "configuration": "Validation; descriptive", "score": 0.889, "raw_score": 88.9, "raw_scale": 100, "source_record": "o1 (high)|Validation; descriptive" }, { "id": "3191c103234745c55db3", "model": "GPT 4.1 mini", "organization": "OpenAI", "configuration": "Validation; descriptive", "score": 0.884, "raw_score": 88.4, "raw_scale": 100, "source_record": "GPT 4.1 mini|Validation; descriptive" }, { "id": "95683ca3059bbecc9565", "model": "GPT 4.1", "organization": "OpenAI", "configuration": "Validation; descriptive", "score": 0.879, "raw_score": 87.9, "raw_scale": 100, "source_record": "GPT 4.1|Validation; descriptive" }, { "id": "ccb8b1669b5654cb4d5c", "model": "o1", "organization": "OpenAI", "configuration": "Validation; descriptive", "score": 0.8745, "raw_score": 87.45, "raw_scale": 100, "source_record": "o1|Validation; descriptive" }, { "id": "7336e89f46e75f8c124d", "model": "Qwen2.5-VL 72B", "organization": "Alibaba", "configuration": "Validation; descriptive", "score": 0.8740000000000001, "raw_score": 87.4, "raw_scale": 100, "source_record": "Qwen2.5-VL 72B|Validation; descriptive" }, { "id": "b4d92d581577b157831b", "model": "InternVL3 38B", "organization": "", "configuration": "Validation; descriptive", "score": 0.872, "raw_score": 87.2, "raw_scale": 100, "source_record": "InternVL3 38B|Validation; descriptive" }, { "id": "3d8a8dbaee5ae1c96f41", "model": "InternVL3 78B", "organization": "", "configuration": "Validation; descriptive", "score": 0.851, "raw_score": 85.1, "raw_scale": 100, "source_record": "InternVL3 78B|Validation; descriptive" }, { "id": "0b116f4f956b9fbf4818", "model": "GPT-4o 240513", "organization": "OpenAI", "configuration": "Validation; descriptive", "score": 0.8445, "raw_score": 84.45, "raw_scale": 100, "source_record": "GPT-4o 240513|Validation; descriptive" }, { "id": "f9649e3d27fba33b9260", "model": "Claude 3.5 Sonnet", "organization": "Anthropic", "configuration": "Validation; descriptive", "score": 0.843, "raw_score": 84.3, "raw_scale": 100, "source_record": "Claude 3.5 Sonnet|Validation; descriptive" }, { "id": "d8afa785893d4b35b0f9", "model": "Doubao 1.5 Pro", "organization": "", "configuration": "Validation; descriptive", "score": 0.843, "raw_score": 84.3, "raw_scale": 100, "source_record": "Doubao 1.5 Pro|Validation; descriptive" }, { "id": "3078362267c5c39fdab5", "model": "InternVL V2.5 78B", "organization": "", "configuration": "Validation; descriptive", "score": 0.823, "raw_score": 82.3, "raw_scale": 100, "source_record": "InternVL V2.5 78B|Validation; descriptive" }, { "id": "5dfbd8f583064a9100d6", "model": "InternVL3 14B", "organization": "", "configuration": "Validation; descriptive", "score": 0.8220000000000001, "raw_score": 82.2, "raw_scale": 100, "source_record": "InternVL3 14B|Validation; descriptive" }, { "id": "58ae16945d46dd45893a", "model": "Qwen2-VL 72B", "organization": "Alibaba", "configuration": "Validation; descriptive", "score": 0.8134999999999999, "raw_score": 81.35, "raw_scale": 100, "source_record": "Qwen2-VL 72B|Validation; descriptive" }, { "id": "a0315a8e576e937300d1", "model": "GPT-4V", "organization": "OpenAI", "configuration": "Validation; descriptive", "score": 0.7992, "raw_score": 79.92, "raw_scale": 100, "source_record": "GPT-4V|Validation; descriptive" }, { "id": "e3e38feadea1999093f8", "model": "InternVL V2.5 38B", "organization": "", "configuration": "Validation; descriptive", "score": 0.7959999999999999, "raw_score": 79.6, "raw_scale": 100, "source_record": "InternVL V2.5 38B|Validation; descriptive" }, { "id": "5a7acc4db660af967f23", "model": "InternVL Chat V2.0 Pro", "organization": "", "configuration": "Validation; descriptive", "score": 0.7683, "raw_score": 76.83, "raw_scale": 100, "source_record": "InternVL Chat V2.0 Pro|Validation; descriptive" }, { "id": "8dbac84f2731be0552dd", "model": "Molmo 72B", "organization": "", "configuration": "Validation; descriptive", "score": 0.7545000000000001, "raw_score": 75.45, "raw_scale": 100, "source_record": "Molmo 72B|Validation; descriptive" }, { "id": "cf7c572e075c08fffa3c", "model": "InternVL Chat V2.0 76B", "organization": "", "configuration": "Validation; descriptive", "score": 0.7517, "raw_score": 75.17, "raw_scale": 100, "source_record": "InternVL Chat V2.0 76B|Validation; descriptive" }, { "id": "9b9a327e654a359b14b2", "model": "GPT-4o Mini", "organization": "OpenAI", "configuration": "Validation; descriptive", "score": 0.7492, "raw_score": 74.92, "raw_scale": 100, "source_record": "GPT-4o Mini|Validation; descriptive" }, { "id": "fd24b980a52cb528b51b", "model": "GPT 4.1 nano", "organization": "OpenAI", "configuration": "Validation; descriptive", "score": 0.7390000000000001, "raw_score": 73.9, "raw_scale": 100, "source_record": "GPT 4.1 nano|Validation; descriptive" }, { "id": "eea169f4132debad11f4", "model": "Qwen2.5-VL 7B", "organization": "Alibaba", "configuration": "Validation; descriptive", "score": 0.7390000000000001, "raw_score": 73.9, "raw_scale": 100, "source_record": "Qwen2.5-VL 7B|Validation; descriptive" }, { "id": "a4a6695706ab76234b06", "model": "Claude 3 Sonnet", "organization": "Anthropic", "configuration": "Validation; descriptive", "score": 0.7365, "raw_score": 73.65, "raw_scale": 100, "source_record": "Claude 3 Sonnet|Validation; descriptive" }, { "id": "b8c865bd76296f9a12ad", "model": "InternVL3 8B", "organization": "", "configuration": "Validation; descriptive", "score": 0.736, "raw_score": 73.6, "raw_scale": 100, "source_record": "InternVL3 8B|Validation; descriptive" }, { "id": "4242b75d4ae2b67cc715", "model": "InternVL V2.5 26B", "organization": "", "configuration": "Validation; descriptive", "score": 0.735, "raw_score": 73.5, "raw_scale": 100, "source_record": "InternVL V2.5 26B|Validation; descriptive" }, { "id": "42f5855ec773068eba54", "model": "InternVL3 9B", "organization": "", "configuration": "Validation; descriptive", "score": 0.725, "raw_score": 72.5, "raw_scale": 100, "source_record": "InternVL3 9B|Validation; descriptive" }, { "id": "294aede2ebbd2208b557", "model": "Gemini 1.5 Pro", "organization": "Google", "configuration": "Validation; descriptive", "score": 0.7197, "raw_score": 71.97, "raw_scale": 100, "source_record": "Gemini 1.5 Pro|Validation; descriptive" }, { "id": "1f07c673a60ee23ab56d", "model": "Llava OneVision 72B", "organization": "", "configuration": "Validation; descriptive", "score": 0.7182999999999999, "raw_score": 71.83, "raw_scale": 100, "source_record": "Llava OneVision 72B|Validation; descriptive" }, { "id": "da0a13dcdb8054a73e46", "model": "Claude 3 Opus", "organization": "Anthropic", "configuration": "Validation; descriptive", "score": 0.7155, "raw_score": 71.55, "raw_scale": 100, "source_record": "Claude 3 Opus|Validation; descriptive" }, { "id": "8dfab8e8606fec7445cd", "model": "InternVL V2.5 8B", "organization": "", "configuration": "Validation; descriptive", "score": 0.6859999999999999, "raw_score": 68.6, "raw_scale": 100, "source_record": "InternVL V2.5 8B|Validation; descriptive" }, { "id": "f689218ddfefee6dfab0", "model": "Llama 3.2 Vision 90B", "organization": "Meta", "configuration": "Validation; descriptive", "score": 0.6842, "raw_score": 68.42, "raw_scale": 100, "source_record": "Llama 3.2 Vision 90B|Validation; descriptive" }, { "id": "0b55f91285b57978c701", "model": "Pixtral 12B", "organization": "", "configuration": "Validation; descriptive", "score": 0.6812, "raw_score": 68.12, "raw_scale": 100, "source_record": "Pixtral 12B|Validation; descriptive" }, { "id": "91498f867afd813f4c3a", "model": "Claude 3 Haiku", "organization": "Anthropic", "configuration": "Validation; descriptive", "score": 0.6507999999999999, "raw_score": 65.08, "raw_scale": 100, "source_record": "Claude 3 Haiku|Validation; descriptive" }, { "id": "56cfb8bc3d3b8533cacd", "model": "NVLM-D 72B", "organization": "", "configuration": "Validation; descriptive", "score": 0.6392, "raw_score": 63.92, "raw_scale": 100, "source_record": "NVLM-D 72B|Validation; descriptive" }, { "id": "4b6fc45f35fde36016dc", "model": "Ovis 1.5 Gemma2 9B", "organization": "", "configuration": "Validation; descriptive", "score": 0.626, "raw_score": 62.6, "raw_scale": 100, "source_record": "Ovis 1.5 Gemma2 9B|Validation; descriptive" }, { "id": "a578693332fbeadec192", "model": "InternVL Chat V2.0 26B", "organization": "", "configuration": "Validation; descriptive", "score": 0.624, "raw_score": 62.4, "raw_scale": 100, "source_record": "InternVL Chat V2.0 26B|Validation; descriptive" }, { "id": "185d349784e4c6ae8b4a", "model": "InternVL V2.5 4B", "organization": "", "configuration": "Validation; descriptive", "score": 0.617, "raw_score": 61.7, "raw_scale": 100, "source_record": "InternVL V2.5 4B|Validation; descriptive" }, { "id": "8c3a16a508398081a5a1", "model": "MiniCPM-V2.6 (Upsize)", "organization": "", "configuration": "Validation; descriptive", "score": 0.6162, "raw_score": 61.62, "raw_scale": 100, "source_record": "MiniCPM-V2.6 (Upsize)|Validation; descriptive" }, { "id": "37ffeccbbb47fe36e7ca", "model": "Phi-3 Vision", "organization": "Microsoft", "configuration": "Validation; descriptive", "score": 0.6048, "raw_score": 60.48, "raw_scale": 100, "source_record": "Phi-3 Vision|Validation; descriptive" }, { "id": "4164bc67bdd90c137aa7", "model": "Ovis 1.5 Llama3 8B", "organization": "", "configuration": "Validation; descriptive", "score": 0.6015, "raw_score": 60.15, "raw_scale": 100, "source_record": "Ovis 1.5 Llama3 8B|Validation; descriptive" }, { "id": "a962ec2da55518369d70", "model": "Cambrian 34B", "organization": "", "configuration": "Validation; descriptive", "score": 0.5972999999999999, "raw_score": 59.73, "raw_scale": 100, "source_record": "Cambrian 34B|Validation; descriptive" }, { "id": "4f4cc380a9325a5b1e74", "model": "Llama 3.2 Vision 11B", "organization": "Meta", "configuration": "Validation; descriptive", "score": 0.5935, "raw_score": 59.35, "raw_scale": 100, "source_record": "Llama 3.2 Vision 11B|Validation; descriptive" }, { "id": "7bd14a7c3dba89800bab", "model": "MiniCPM-V2.5", "organization": "", "configuration": "Validation; descriptive", "score": 0.5927, "raw_score": 59.27, "raw_scale": 100, "source_record": "MiniCPM-V2.5|Validation; descriptive" }, { "id": "4d82a085b3f4c5136880", "model": "Qwen2.5-VL 3B", "organization": "Alibaba", "configuration": "Validation; descriptive", "score": 0.586, "raw_score": 58.6, "raw_scale": 100, "source_record": "Qwen2.5-VL 3B|Validation; descriptive" }, { "id": "fc258c1fb542a1387d84", "model": "InternVL Chat V1.5", "organization": "", "configuration": "Validation; descriptive", "score": 0.585, "raw_score": 58.5, "raw_scale": 100, "source_record": "InternVL Chat V1.5|Validation; descriptive" }, { "id": "d2f958c340ebf2ec5be1", "model": "Qwen2-VL 7B", "organization": "Alibaba", "configuration": "Validation; descriptive", "score": 0.58, "raw_score": 58.0, "raw_scale": 100, "source_record": "Qwen2-VL 7B|Validation; descriptive" }, { "id": "481ce559f52bf4bb4c3a", "model": "GLM 4V 9B", "organization": "Z AI", "configuration": "Validation; descriptive", "score": 0.5761999999999999, "raw_score": 57.62, "raw_scale": 100, "source_record": "GLM 4V 9B|Validation; descriptive" }, { "id": "7483dc978516b24d6461", "model": "MiniCPM-V2.6 (Upsize+CoT)", "organization": "", "configuration": "Validation; descriptive", "score": 0.5705, "raw_score": 57.05, "raw_scale": 100, "source_record": "MiniCPM-V2.6 (Upsize+CoT)|Validation; descriptive" }, { "id": "dbf8d95b3eca15617840", "model": "Reka Flash", "organization": "", "configuration": "Validation; descriptive", "score": 0.5645, "raw_score": 56.45, "raw_scale": 100, "source_record": "Reka Flash|Validation; descriptive" }, { "id": "670312f298a6c7acc630", "model": "Reka Core", "organization": "", "configuration": "Validation; descriptive", "score": 0.556, "raw_score": 55.6, "raw_scale": 100, "source_record": "Reka Core|Validation; descriptive" }, { "id": "d2cf298de7dbbbdb7d8b", "model": "Phi-3.5 Vision", "organization": "Microsoft", "configuration": "Validation; descriptive", "score": 0.5502, "raw_score": 55.02, "raw_scale": 100, "source_record": "Phi-3.5 Vision|Validation; descriptive" }, { "id": "c58152ba461b6a1774dc", "model": "InternVL3 2B", "organization": "", "configuration": "Validation; descriptive", "score": 0.547, "raw_score": 54.7, "raw_scale": 100, "source_record": "InternVL3 2B|Validation; descriptive" }, { "id": "102907aa792bd5d95ffa", "model": "InternLM XComposer2 4KHD", "organization": "Shanghai AI Laboratory", "configuration": "Validation; descriptive", "score": 0.5465, "raw_score": 54.65, "raw_scale": 100, "source_record": "InternLM XComposer2 4KHD|Validation; descriptive" }, { "id": "08eeddce7e2231655f18", "model": "Gemini 1.0 Pro", "organization": "Google", "configuration": "Validation; descriptive", "score": 0.5437, "raw_score": 54.37, "raw_scale": 100, "source_record": "Gemini 1.0 Pro|Validation; descriptive" }, { "id": "22153ff4872c5643dd9d", "model": "Molmo 7B D", "organization": "", "configuration": "Validation; descriptive", "score": 0.5273, "raw_score": 52.73, "raw_scale": 100, "source_record": "Molmo 7B D|Validation; descriptive" }, { "id": "065951eb87947fa96229", "model": "Mini Gemini HD Yi 34B", "organization": "", "configuration": "Validation; descriptive", "score": 0.5268, "raw_score": 52.68, "raw_scale": 100, "source_record": "Mini Gemini HD Yi 34B|Validation; descriptive" }, { "id": "14fb09da390a60f2883c", "model": "LLaVA 1.6 Yi 34B", "organization": "", "configuration": "Validation; descriptive", "score": 0.5105, "raw_score": 51.05, "raw_scale": 100, "source_record": "LLaVA 1.6 Yi 34B|Validation; descriptive" }, { "id": "0b65e7f8fd2767da8d40", "model": "InternVL V2.5 2B", "organization": "", "configuration": "Validation; descriptive", "score": 0.49700000000000005, "raw_score": 49.7, "raw_scale": 100, "source_record": "InternVL V2.5 2B|Validation; descriptive" }, { "id": "f04312d8566aa257fd1f", "model": "Llava OneVision 7B", "organization": "", "configuration": "Validation; descriptive", "score": 0.48729999999999996, "raw_score": 48.73, "raw_scale": 100, "source_record": "Llava OneVision 7B|Validation; descriptive" }, { "id": "7794a0d584363ff4559e", "model": "InternVL3 1B", "organization": "", "configuration": "Validation; descriptive", "score": 0.47100000000000003, "raw_score": 47.1, "raw_scale": 100, "source_record": "InternVL3 1B|Validation; descriptive" }, { "id": "6f5f199ea4cff76797eb", "model": "DeepSeek VL", "organization": "DeepSeek", "configuration": "Validation; descriptive", "score": 0.45799999999999996, "raw_score": 45.8, "raw_scale": 100, "source_record": "DeepSeek VL|Validation; descriptive" }, { "id": "ac900a51bd05f7c2aa8d", "model": "Molmo 7B O", "organization": "", "configuration": "Validation; descriptive", "score": 0.4567, "raw_score": 45.67, "raw_scale": 100, "source_record": "Molmo 7B O|Validation; descriptive" }, { "id": "645bd85957647e75eadd", "model": "Qwen2-VL 2B", "organization": "Alibaba", "configuration": "Validation; descriptive", "score": 0.44549999999999995, "raw_score": 44.55, "raw_scale": 100, "source_record": "Qwen2-VL 2B|Validation; descriptive" }, { "id": "fdc031d6e8e8c742ddba", "model": "Mini Gemini HD Llama3 8B", "organization": "", "configuration": "Validation; descriptive", "score": 0.44420000000000004, "raw_score": 44.42, "raw_scale": 100, "source_record": "Mini Gemini HD Llama3 8B|Validation; descriptive" }, { "id": "fa4e61efdd65ae18416c", "model": "IDEFICS 2 Chatty", "organization": "", "configuration": "Validation; descriptive", "score": 0.4155, "raw_score": 41.55, "raw_scale": 100, "source_record": "IDEFICS 2 Chatty|Validation; descriptive" }, { "id": "18e6a154d00054ad03c9", "model": "Qwen VL Max", "organization": "Alibaba", "configuration": "Validation; descriptive", "score": 0.41479999999999995, "raw_score": 41.48, "raw_scale": 100, "source_record": "Qwen VL Max|Validation; descriptive" }, { "id": "98efab2b57018a1bc125", "model": "MolmoE 1B", "organization": "", "configuration": "Validation; descriptive", "score": 0.408, "raw_score": 40.8, "raw_scale": 100, "source_record": "MolmoE 1B|Validation; descriptive" }, { "id": "12c377b555f9e87d8ee0", "model": "InternLM XComposer2", "organization": "Shanghai AI Laboratory", "configuration": "Validation; descriptive", "score": 0.3875, "raw_score": 38.75, "raw_scale": 100, "source_record": "InternLM XComposer2|Validation; descriptive" }, { "id": "edf8467b731ed12599b4", "model": "VILA 1.5 40B", "organization": "", "configuration": "Validation; descriptive", "score": 0.38670000000000004, "raw_score": 38.67, "raw_scale": 100, "source_record": "VILA 1.5 40B|Validation; descriptive" }, { "id": "f3b29b94b76a42f2a95a", "model": "InternVL V2.5 1B", "organization": "", "configuration": "Validation; descriptive", "score": 0.384, "raw_score": 38.4, "raw_scale": 100, "source_record": "InternVL V2.5 1B|Validation; descriptive" }, { "id": "7e601328adb95cdecd3d", "model": "DocOwl 1.5 Chat", "organization": "", "configuration": "Validation; descriptive", "score": 0.374, "raw_score": 37.4, "raw_scale": 100, "source_record": "DocOwl 1.5 Chat|Validation; descriptive" }, { "id": "d439c9c3c93cb050508a", "model": "CogAgent", "organization": "", "configuration": "Validation; descriptive", "score": 0.363, "raw_score": 36.3, "raw_scale": 100, "source_record": "CogAgent|Validation; descriptive" }, { "id": "b474896f33d89ae3617d", "model": "MiniCPM-V2", "organization": "", "configuration": "Validation; descriptive", "score": 0.3577, "raw_score": 35.77, "raw_scale": 100, "source_record": "MiniCPM-V2|Validation; descriptive" }, { "id": "9ca241acbca048d99581", "model": "LLaVA 1.6 Mistral 7B", "organization": "", "configuration": "Validation; descriptive", "score": 0.354, "raw_score": 35.4, "raw_scale": 100, "source_record": "LLaVA 1.6 Mistral 7B|Validation; descriptive" }, { "id": "137e5b3e8d19c123bf1d", "model": "Reka Edge", "organization": "", "configuration": "Validation; descriptive", "score": 0.33649999999999997, "raw_score": 33.65, "raw_scale": 100, "source_record": "Reka Edge|Validation; descriptive" }, { "id": "854311a79577c989800e", "model": "IDEFICS 2", "organization": "", "configuration": "Validation; descriptive", "score": 0.32770000000000005, "raw_score": 32.77, "raw_scale": 100, "source_record": "IDEFICS 2|Validation; descriptive" }, { "id": "27820a40bc261db4a452", "model": "Llava OneVision 0.5B", "organization": "", "configuration": "Validation; descriptive", "score": 0.32770000000000005, "raw_score": 32.77, "raw_scale": 100, "source_record": "Llava OneVision 0.5B|Validation; descriptive" }, { "id": "8afc69846487e43e2266", "model": "SPHINX V2", "organization": "", "configuration": "Validation; descriptive", "score": 0.3025, "raw_score": 30.25, "raw_scale": 100, "source_record": "SPHINX V2|Validation; descriptive" }, { "id": "b53f291324c64d7f1fcd", "model": "Qwen VL Plus", "organization": "Alibaba", "configuration": "Validation; descriptive", "score": 0.2893, "raw_score": 28.93, "raw_scale": 100, "source_record": "Qwen VL Plus|Validation; descriptive" }, { "id": "d7deafe35dde133314d6", "model": "MoAI", "organization": "", "configuration": "Validation; descriptive", "score": 0.287, "raw_score": 28.7, "raw_scale": 100, "source_record": "MoAI|Validation; descriptive" }, { "id": "daccea2bcd00008ca6c8", "model": "DocOwl 1.5 Omni", "organization": "", "configuration": "Validation; descriptive", "score": 0.257, "raw_score": 25.7, "raw_scale": 100, "source_record": "DocOwl 1.5 Omni|Validation; descriptive" }, { "id": "40aa672d739cf462504c", "model": "ChartInstruct-Llama2", "organization": "", "configuration": "Validation; descriptive", "score": 0.214, "raw_score": 21.4, "raw_scale": 100, "source_record": "ChartInstruct-Llama2|Validation; descriptive" }, { "id": "1c9e0bb5f32f9c170fc6", "model": "ChartGemma", "organization": "", "configuration": "Validation; descriptive", "score": 0.213, "raw_score": 21.3, "raw_scale": 100, "source_record": "ChartGemma|Validation; descriptive" }, { "id": "4f1750b0c2345793d551", "model": "UniChart-ChartQA", "organization": "", "configuration": "Validation; descriptive", "score": 0.1932, "raw_score": 19.32, "raw_scale": 100, "source_record": "UniChart-ChartQA|Validation; descriptive" }, { "id": "bf11503dc43f1f2778cf", "model": "ChartLlama", "organization": "", "configuration": "Validation; descriptive", "score": 0.1923, "raw_score": 19.23, "raw_scale": 100, "source_record": "ChartLlama|Validation; descriptive" }, { "id": "f7145f53d49f60ea2f70", "model": "UReader", "organization": "", "configuration": "Validation; descriptive", "score": 0.1898, "raw_score": 18.98, "raw_scale": 100, "source_record": "UReader|Validation; descriptive" }, { "id": "d3af2a7f01a2eb4e7bcb", "model": "ChartAssistant", "organization": "", "configuration": "Validation; descriptive", "score": 0.1693, "raw_score": 16.93, "raw_scale": 100, "source_record": "ChartAssistant|Validation; descriptive" }, { "id": "bd365c1c4f6bd12502c7", "model": "TinyChart", "organization": "", "configuration": "Validation; descriptive", "score": 0.16149999999999998, "raw_score": 16.15, "raw_scale": 100, "source_record": "TinyChart|Validation; descriptive" }, { "id": "95b8bb72c612afd9205e", "model": "ChartInstruct-FlanT5", "organization": "", "configuration": "Validation; descriptive", "score": 0.1547, "raw_score": 15.47, "raw_scale": 100, "source_record": "ChartInstruct-FlanT5|Validation; descriptive" }, { "id": "7cafe965037d9d95dd50", "model": "TextMonkey", "organization": "", "configuration": "Validation; descriptive", "score": 0.1245, "raw_score": 12.45, "raw_scale": 100, "source_record": "TextMonkey|Validation; descriptive" } ] }, { "id": "ocrbench-v2-en", "name": "OCRBench v2 (English)", "group": "视觉与视频", "unit": "score", "metric": "Average task score", "protocol": { "en": "Official language-specific task average on a 0–100 scale. Heterogeneous task metrics are retained as a composite score, outside accuracy saturation groups.", "zh": "对应语言各任务的官方平均分，保留 0–100 综合分数；各任务计分不同，不参与准确率饱和分组。" }, "source": { "data_url": "https://huggingface.co/spaces/ling99/OCRBench-v2-leaderboard/resolve/main/OCRBench_en.csv", "sha256": "7544c265a28b76dc8db36dd762aa839685fe367a739413a631a35a895b3ecbbe", "retrieved_at": "2026-11-01T16:40:34.818255+00:00", "name": "OCRBench v2", "url": "https://huggingface.co/spaces/ling99/OCRBench-v2-leaderboard", "selector": "Average Score; original 0–100 task-average scale; not a single accuracy metric", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "dac52f0031735b2a7e11", "model": "Llama Nemotron Nano VL 8B", "organization": "Meta", "configuration": "Published task-average score (0–100)", "score": 60.2, "raw_score": 60.2, "raw_scale": 1, "source_record": "Llama Nemotron Nano VL 8B|Published task-average score (0–100)" }, { "id": "946dcc34687c67d255b7", "model": "InternVL-3-14B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 52.6, "raw_score": 52.6, "raw_scale": 1, "source_record": "InternVL-3-14B|Published task-average score (0–100)" }, { "id": "0773abda1becfe568dd8", "model": "Gemini-Pro", "organization": "Google", "configuration": "Published task-average score (0–100)", "score": 51.9, "raw_score": 51.9, "raw_scale": 1, "source_record": "Gemini-Pro|Published task-average score (0–100)" }, { "id": "3b4ba5bdfea57bb848c5", "model": "Qwen2-VL-7B", "organization": "Alibaba", "configuration": "Published task-average score (0–100)", "score": 51.4, "raw_score": 51.4, "raw_scale": 1, "source_record": "Qwen2-VL-7B|Published task-average score (0–100)" }, { "id": "c7f690db7497aa77b7ac", "model": "InternVL2.5-26B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 49.4, "raw_score": 49.4, "raw_scale": 1, "source_record": "InternVL2.5-26B|Published task-average score (0–100)" }, { "id": "a770ea53a91eb9aa0522", "model": "InternVL-3-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 49.0, "raw_score": 49.0, "raw_scale": 1, "source_record": "InternVL-3-8B|Published task-average score (0–100)" }, { "id": "3187c639136e98ddd89d", "model": "GPT-4V", "organization": "OpenAI", "configuration": "Published task-average score (0–100)", "score": 47.7, "raw_score": 47.7, "raw_scale": 1, "source_record": "GPT-4V|Published task-average score (0–100)" }, { "id": "0b8cb711aa4bc62cd2f7", "model": "Ovis2-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 47.7, "raw_score": 47.7, "raw_scale": 1, "source_record": "Ovis2-8B|Published task-average score (0–100)" }, { "id": "ab42ff0b7ed004f9252c", "model": "InternVL2-26B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 46.8, "raw_score": 46.8, "raw_scale": 1, "source_record": "InternVL2-26B|Published task-average score (0–100)" }, { "id": "f086112b6b38986f0f1d", "model": "Qwen2.5-VL-7B", "organization": "Alibaba", "configuration": "Published task-average score (0–100)", "score": 46.7, "raw_score": 46.7, "raw_scale": 1, "source_record": "Qwen2.5-VL-7B|Published task-average score (0–100)" }, { "id": "a8cae5480c6ed635686b", "model": "Step-1V", "organization": "", "configuration": "Published task-average score (0–100)", "score": 46.7, "raw_score": 46.7, "raw_scale": 1, "source_record": "Step-1V|Published task-average score (0–100)" }, { "id": "3d64f67625abaa613a0d", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Published task-average score (0–100)", "score": 46.5, "raw_score": 46.5, "raw_scale": 1, "source_record": "GPT-4o|Published task-average score (0–100)" }, { "id": "0ca19dfc678e9cad49f6", "model": "Claude3.5-sonnet", "organization": "Anthropic", "configuration": "Published task-average score (0–100)", "score": 45.2, "raw_score": 45.2, "raw_scale": 1, "source_record": "Claude3.5-sonnet|Published task-average score (0–100)" }, { "id": "d6e63a301bd6e8aef734", "model": "MiniCPM-o-2.6", "organization": "", "configuration": "Published task-average score (0–100)", "score": 45.1, "raw_score": 45.1, "raw_scale": 1, "source_record": "MiniCPM-o-2.6|Published task-average score (0–100)" }, { "id": "943858ce9ba501a47170", "model": "InternVL2.5-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 45.0, "raw_score": 45.0, "raw_scale": 1, "source_record": "InternVL2.5-8B|Published task-average score (0–100)" }, { "id": "e508dcb29660a6759967", "model": "SAIL-VL-1.6-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 44.8, "raw_score": 44.8, "raw_scale": 1, "source_record": "SAIL-VL-1.6-8B|Published task-average score (0–100)" }, { "id": "b530031ab5c006110a7d", "model": "Deepseek-VL2-Small", "organization": "DeepSeek", "configuration": "Published task-average score (0–100)", "score": 43.3, "raw_score": 43.3, "raw_scale": 1, "source_record": "Deepseek-VL2-Small|Published task-average score (0–100)" }, { "id": "f51d8a4c1e55137e9f6f", "model": "GPT-4o-mini", "organization": "OpenAI", "configuration": "Published task-average score (0–100)", "score": 43.0, "raw_score": 43.0, "raw_scale": 1, "source_record": "GPT-4o-mini|Published task-average score (0–100)" }, { "id": "c934c479d202276f7de3", "model": "GLM-4V-Plus", "organization": "Z AI", "configuration": "Published task-average score (0–100)", "score": 42.9, "raw_score": 42.9, "raw_scale": 1, "source_record": "GLM-4V-Plus|Published task-average score (0–100)" }, { "id": "c77ae4569203f4f7c5b3", "model": "GLM-4V-9B", "organization": "Z AI", "configuration": "Published task-average score (0–100)", "score": 42.6, "raw_score": 42.6, "raw_scale": 1, "source_record": "GLM-4V-9B|Published task-average score (0–100)" }, { "id": "d16edd04c93f12665cd8", "model": "Kimi-VL-A3-16B", "organization": "Moonshot AI", "configuration": "Published task-average score (0–100)", "score": 41.1, "raw_score": 41.1, "raw_scale": 1, "source_record": "Kimi-VL-A3-16B|Published task-average score (0–100)" }, { "id": "78de0c4035e245b639e4", "model": "Ovis1.6-3B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 40.8, "raw_score": 40.8, "raw_scale": 1, "source_record": "Ovis1.6-3B|Published task-average score (0–100)" }, { "id": "9f5046c4a0a7e6ec89b4", "model": "MiniCPM-V-2.6", "organization": "", "configuration": "Published task-average score (0–100)", "score": 40.3, "raw_score": 40.3, "raw_scale": 1, "source_record": "MiniCPM-V-2.6|Published task-average score (0–100)" }, { "id": "2a77aa97a77a5e1cbe6f", "model": "Pixtral-12B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 40.3, "raw_score": 40.3, "raw_scale": 1, "source_record": "Pixtral-12B|Published task-average score (0–100)" }, { "id": "20b525065971ebb1d8c4", "model": "InternVL2-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 39.6, "raw_score": 39.6, "raw_scale": 1, "source_record": "InternVL2-8B|Published task-average score (0–100)" }, { "id": "2519cc850bff7759cc80", "model": "LLaVA-OV-7B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 36.4, "raw_score": 36.4, "raw_scale": 1, "source_record": "LLaVA-OV-7B|Published task-average score (0–100)" }, { "id": "2d0b9f49adfb7d1eeebd", "model": "Phi-4-MultiModal", "organization": "Microsoft", "configuration": "Published task-average score (0–100)", "score": 35.2, "raw_score": 35.2, "raw_scale": 1, "source_record": "Phi-4-MultiModal|Published task-average score (0–100)" }, { "id": "29392c57e63e7fb91990", "model": "Cambrian-1-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 34.7, "raw_score": 34.7, "raw_scale": 1, "source_record": "Cambrian-1-8B|Published task-average score (0–100)" }, { "id": "fc43d21c669fe1e4732d", "model": "Molmo-7B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 34.5, "raw_score": 34.5, "raw_scale": 1, "source_record": "Molmo-7B|Published task-average score (0–100)" }, { "id": "51403000e8ff1c4e9daa", "model": "Idefics3-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 32.2, "raw_score": 32.2, "raw_scale": 1, "source_record": "Idefics3-8B|Published task-average score (0–100)" }, { "id": "40de65ed37dbab95d945", "model": "LLaVA-Next-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 31.5, "raw_score": 31.5, "raw_scale": 1, "source_record": "LLaVA-Next-8B|Published task-average score (0–100)" }, { "id": "e096e8f41f0044c1af13", "model": "XComposer2-4KHD", "organization": "", "configuration": "Published task-average score (0–100)", "score": 27.9, "raw_score": 27.9, "raw_scale": 1, "source_record": "XComposer2-4KHD|Published task-average score (0–100)" }, { "id": "bab39736cb65a2bd07be", "model": "Eagle-X5-7B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 27.5, "raw_score": 27.5, "raw_scale": 1, "source_record": "Eagle-X5-7B|Published task-average score (0–100)" }, { "id": "7fc2a164d8815ab6b412", "model": "Deepseek-VL-7B", "organization": "DeepSeek", "configuration": "Published task-average score (0–100)", "score": 27.2, "raw_score": 27.2, "raw_scale": 1, "source_record": "Deepseek-VL-7B|Published task-average score (0–100)" }, { "id": "c3b3567eb06d725bcedd", "model": "mPLUG-Owl3", "organization": "", "configuration": "Published task-average score (0–100)", "score": 25.1, "raw_score": 25.1, "raw_scale": 1, "source_record": "mPLUG-Owl3|Published task-average score (0–100)" }, { "id": "8a37c06d42702ed1188f", "model": "TextMonkey", "organization": "", "configuration": "Published task-average score (0–100)", "score": 23.9, "raw_score": 23.9, "raw_scale": 1, "source_record": "TextMonkey|Published task-average score (0–100)" }, { "id": "d446f453bcc5c6f5cf0b", "model": "VILA1.5-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 23.6, "raw_score": 23.6, "raw_scale": 1, "source_record": "VILA1.5-8B|Published task-average score (0–100)" }, { "id": "ea9bab8a2cebcbf6fa13", "model": "Qwen-VL-chat", "organization": "Alibaba", "configuration": "Published task-average score (0–100)", "score": 23.4, "raw_score": 23.4, "raw_scale": 1, "source_record": "Qwen-VL-chat|Published task-average score (0–100)" }, { "id": "bcfd2234038a3e810fac", "model": "Qwen-VL", "organization": "Alibaba", "configuration": "Published task-average score (0–100)", "score": 23.3, "raw_score": 23.3, "raw_scale": 1, "source_record": "Qwen-VL|Published task-average score (0–100)" }, { "id": "4a7f92278f91e28b77d9", "model": "Monkey", "organization": "", "configuration": "Published task-average score (0–100)", "score": 23.1, "raw_score": 23.1, "raw_scale": 1, "source_record": "Monkey|Published task-average score (0–100)" }, { "id": "2d07c8a080894b999f41", "model": "CogVLM-chat", "organization": "", "configuration": "Published task-average score (0–100)", "score": 21.8, "raw_score": 21.8, "raw_scale": 1, "source_record": "CogVLM-chat|Published task-average score (0–100)" }, { "id": "909cbea6eee7e6f3c418", "model": "DocOwl2", "organization": "", "configuration": "Published task-average score (0–100)", "score": 19.4, "raw_score": 19.4, "raw_scale": 1, "source_record": "DocOwl2|Published task-average score (0–100)" }, { "id": "432d9ac1f7fb97ab26f3", "model": "EMU2-chat", "organization": "", "configuration": "Published task-average score (0–100)", "score": 18.8, "raw_score": 18.8, "raw_scale": 1, "source_record": "EMU2-chat|Published task-average score (0–100)" }, { "id": "d89c603b41ae655d1b5c", "model": "Janus-1.3B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 18.7, "raw_score": 18.7, "raw_scale": 1, "source_record": "Janus-1.3B|Published task-average score (0–100)" }, { "id": "7d7ed0f8ecc7af91c4f5", "model": "Yi-VL-6B", "organization": "01.AI", "configuration": "Published task-average score (0–100)", "score": 17.3, "raw_score": 17.3, "raw_scale": 1, "source_record": "Yi-VL-6B|Published task-average score (0–100)" }, { "id": "5533541e15b96bd6546c", "model": "TextHarmony", "organization": "", "configuration": "Published task-average score (0–100)", "score": 16.0, "raw_score": 16.0, "raw_scale": 1, "source_record": "TextHarmony|Published task-average score (0–100)" }, { "id": "73d40ed0585a5ca446e1", "model": "LLaVAR", "organization": "", "configuration": "Published task-average score (0–100)", "score": 15.3, "raw_score": 15.3, "raw_scale": 1, "source_record": "LLaVAR|Published task-average score (0–100)" }, { "id": "c08e1c1c46b906392831", "model": "UReader", "organization": "", "configuration": "Published task-average score (0–100)", "score": 13.7, "raw_score": 13.7, "raw_scale": 1, "source_record": "UReader|Published task-average score (0–100)" } ] }, { "id": "ocrbench-v2-cn", "name": "OCRBench v2 (Chinese)", "group": "视觉与视频", "unit": "score", "metric": "Average task score", "protocol": { "en": "Official language-specific task average on a 0–100 scale. Heterogeneous task metrics are retained as a composite score, outside accuracy saturation groups.", "zh": "对应语言各任务的官方平均分，保留 0–100 综合分数；各任务计分不同，不参与准确率饱和分组。" }, "source": { "data_url": "https://huggingface.co/spaces/ling99/OCRBench-v2-leaderboard/resolve/main/OCRBench_cn.csv", "sha256": "23272e230347240baa4d42e77185e65e8fbf3fd3e01a1c59fface1bd9cd5f97e", "retrieved_at": "2026-11-01T16:40:34.830742+00:00", "name": "OCRBench v2", "url": "https://huggingface.co/spaces/ling99/OCRBench-v2-leaderboard", "selector": "Average Score; original 0–100 task-average scale; not a single accuracy metric", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "ad7072aa4b2aded7fb15", "model": "InternVL3-14B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 55.7, "raw_score": 55.7, "raw_scale": 1, "source_record": "InternVL3-14B|Published task-average score (0–100)" }, { "id": "f086112b6b38986f0f1d", "model": "Qwen2.5-VL-7B", "organization": "Alibaba", "configuration": "Published task-average score (0–100)", "score": 55.6, "raw_score": 55.6, "raw_scale": 1, "source_record": "Qwen2.5-VL-7B|Published task-average score (0–100)" }, { "id": "788c6119d20f0a288bea", "model": "InternVL3-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 53.5, "raw_score": 53.5, "raw_scale": 1, "source_record": "InternVL3-8B|Published task-average score (0–100)" }, { "id": "0b8cb711aa4bc62cd2f7", "model": "Ovis2-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 49.2, "raw_score": 49.2, "raw_scale": 1, "source_record": "Ovis2-8B|Published task-average score (0–100)" }, { "id": "943858ce9ba501a47170", "model": "InternVL2.5-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 46.2, "raw_score": 46.2, "raw_scale": 1, "source_record": "InternVL2.5-8B|Published task-average score (0–100)" }, { "id": "98c917abd7d4ee538574", "model": "Kimi-VL-A3B-16B", "organization": "Moonshot AI", "configuration": "Published task-average score (0–100)", "score": 45.5, "raw_score": 45.5, "raw_scale": 1, "source_record": "Kimi-VL-A3B-16B|Published task-average score (0–100)" }, { "id": "c7f690db7497aa77b7ac", "model": "InternVL2.5-26B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 44.2, "raw_score": 44.2, "raw_scale": 1, "source_record": "InternVL2.5-26B|Published task-average score (0–100)" }, { "id": "0773abda1becfe568dd8", "model": "Gemini-Pro", "organization": "Google", "configuration": "Published task-average score (0–100)", "score": 43.1, "raw_score": 43.1, "raw_scale": 1, "source_record": "Gemini-Pro|Published task-average score (0–100)" }, { "id": "3b4ba5bdfea57bb848c5", "model": "Qwen2-VL-7B", "organization": "Alibaba", "configuration": "Published task-average score (0–100)", "score": 42.9, "raw_score": 42.9, "raw_scale": 1, "source_record": "Qwen2-VL-7B|Published task-average score (0–100)" }, { "id": "b530031ab5c006110a7d", "model": "Deepseek-VL2-Small", "organization": "DeepSeek", "configuration": "Published task-average score (0–100)", "score": 42.7, "raw_score": 42.7, "raw_scale": 1, "source_record": "Deepseek-VL2-Small|Published task-average score (0–100)" }, { "id": "a8cae5480c6ed635686b", "model": "Step-1V", "organization": "", "configuration": "Published task-average score (0–100)", "score": 42.6, "raw_score": 42.6, "raw_scale": 1, "source_record": "Step-1V|Published task-average score (0–100)" }, { "id": "d6e63a301bd6e8aef734", "model": "MiniCPM-o-2.6", "organization": "", "configuration": "Published task-average score (0–100)", "score": 41.1, "raw_score": 41.1, "raw_scale": 1, "source_record": "MiniCPM-o-2.6|Published task-average score (0–100)" }, { "id": "3187c639136e98ddd89d", "model": "GPT-4V", "organization": "OpenAI", "configuration": "Published task-average score (0–100)", "score": 40.1, "raw_score": 40.1, "raw_scale": 1, "source_record": "GPT-4V|Published task-average score (0–100)" }, { "id": "0ca19dfc678e9cad49f6", "model": "Claude3.5-sonnet", "organization": "Anthropic", "configuration": "Published task-average score (0–100)", "score": 39.6, "raw_score": 39.6, "raw_scale": 1, "source_record": "Claude3.5-sonnet|Published task-average score (0–100)" }, { "id": "c934c479d202276f7de3", "model": "GLM-4V-Plus", "organization": "Z AI", "configuration": "Published task-average score (0–100)", "score": 39.4, "raw_score": 39.4, "raw_scale": 1, "source_record": "GLM-4V-Plus|Published task-average score (0–100)" }, { "id": "dac52f0031735b2a7e11", "model": "Llama Nemotron Nano VL 8B", "organization": "Meta", "configuration": "Published task-average score (0–100)", "score": 37.9, "raw_score": 37.9, "raw_scale": 1, "source_record": "Llama Nemotron Nano VL 8B|Published task-average score (0–100)" }, { "id": "ab42ff0b7ed004f9252c", "model": "InternVL2-26B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 37.7, "raw_score": 37.7, "raw_scale": 1, "source_record": "InternVL2-26B|Published task-average score (0–100)" }, { "id": "c77ae4569203f4f7c5b3", "model": "GLM-4V-9B", "organization": "Z AI", "configuration": "Published task-average score (0–100)", "score": 36.6, "raw_score": 36.6, "raw_scale": 1, "source_record": "GLM-4V-9B|Published task-average score (0–100)" }, { "id": "20b525065971ebb1d8c4", "model": "InternVL2-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 36.3, "raw_score": 36.3, "raw_scale": 1, "source_record": "InternVL2-8B|Published task-average score (0–100)" }, { "id": "e508dcb29660a6759967", "model": "SAIL-VL-1.6-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 34.5, "raw_score": 34.5, "raw_scale": 1, "source_record": "SAIL-VL-1.6-8B|Published task-average score (0–100)" }, { "id": "9f5046c4a0a7e6ec89b4", "model": "MiniCPM-V-2.6", "organization": "", "configuration": "Published task-average score (0–100)", "score": 33.9, "raw_score": 33.9, "raw_scale": 1, "source_record": "MiniCPM-V-2.6|Published task-average score (0–100)" }, { "id": "3d64f67625abaa613a0d", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Published task-average score (0–100)", "score": 32.2, "raw_score": 32.2, "raw_scale": 1, "source_record": "GPT-4o|Published task-average score (0–100)" }, { "id": "2d0b9f49adfb7d1eeebd", "model": "Phi-4-MultiModal", "organization": "Microsoft", "configuration": "Published task-average score (0–100)", "score": 30.7, "raw_score": 30.7, "raw_scale": 1, "source_record": "Phi-4-MultiModal|Published task-average score (0–100)" }, { "id": "f51d8a4c1e55137e9f6f", "model": "GPT-4o-mini", "organization": "OpenAI", "configuration": "Published task-average score (0–100)", "score": 25.0, "raw_score": 25.0, "raw_scale": 1, "source_record": "GPT-4o-mini|Published task-average score (0–100)" }, { "id": "78de0c4035e245b639e4", "model": "Ovis1.6-3B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 21.1, "raw_score": 21.1, "raw_scale": 1, "source_record": "Ovis1.6-3B|Published task-average score (0–100)" }, { "id": "2519cc850bff7759cc80", "model": "LLaVA-OV-7B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 17.8, "raw_score": 17.8, "raw_scale": 1, "source_record": "LLaVA-OV-7B|Published task-average score (0–100)" }, { "id": "8a37c06d42702ed1188f", "model": "TextMonkey", "organization": "", "configuration": "Published task-average score (0–100)", "score": 15.8, "raw_score": 15.8, "raw_scale": 1, "source_record": "TextMonkey|Published task-average score (0–100)" }, { "id": "e096e8f41f0044c1af13", "model": "XComposer2-4KHD", "organization": "", "configuration": "Published task-average score (0–100)", "score": 15.5, "raw_score": 15.5, "raw_scale": 1, "source_record": "XComposer2-4KHD|Published task-average score (0–100)" }, { "id": "2a77aa97a77a5e1cbe6f", "model": "Pixtral-12B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 14.6, "raw_score": 14.6, "raw_scale": 1, "source_record": "Pixtral-12B|Published task-average score (0–100)" }, { "id": "c3b3567eb06d725bcedd", "model": "mPLUG-Owl3", "organization": "", "configuration": "Published task-average score (0–100)", "score": 13.3, "raw_score": 13.3, "raw_scale": 1, "source_record": "mPLUG-Owl3|Published task-average score (0–100)" }, { "id": "51403000e8ff1c4e9daa", "model": "Idefics3-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 13.1, "raw_score": 13.1, "raw_scale": 1, "source_record": "Idefics3-8B|Published task-average score (0–100)" }, { "id": "4a7f92278f91e28b77d9", "model": "Monkey", "organization": "", "configuration": "Published task-average score (0–100)", "score": 13.1, "raw_score": 13.1, "raw_scale": 1, "source_record": "Monkey|Published task-average score (0–100)" }, { "id": "fc43d21c669fe1e4732d", "model": "Molmo-7B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 12.8, "raw_score": 12.8, "raw_scale": 1, "source_record": "Molmo-7B|Published task-average score (0–100)" }, { "id": "7fc2a164d8815ab6b412", "model": "Deepseek-VL-7B", "organization": "DeepSeek", "configuration": "Published task-average score (0–100)", "score": 12.2, "raw_score": 12.2, "raw_scale": 1, "source_record": "Deepseek-VL-7B|Published task-average score (0–100)" }, { "id": "ea9bab8a2cebcbf6fa13", "model": "Qwen-VL-chat", "organization": "Alibaba", "configuration": "Published task-average score (0–100)", "score": 11.8, "raw_score": 11.8, "raw_scale": 1, "source_record": "Qwen-VL-chat|Published task-average score (0–100)" }, { "id": "bab39736cb65a2bd07be", "model": "Eagle-X5-7B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 11.1, "raw_score": 11.1, "raw_scale": 1, "source_record": "Eagle-X5-7B|Published task-average score (0–100)" }, { "id": "29392c57e63e7fb91990", "model": "Cambrian-1-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 9.9, "raw_score": 9.9, "raw_scale": 1, "source_record": "Cambrian-1-8B|Published task-average score (0–100)" }, { "id": "7d7ed0f8ecc7af91c4f5", "model": "Yi-VL-6B", "organization": "01.AI", "configuration": "Published task-average score (0–100)", "score": 9.4, "raw_score": 9.4, "raw_scale": 1, "source_record": "Yi-VL-6B|Published task-average score (0–100)" }, { "id": "bcfd2234038a3e810fac", "model": "Qwen-VL", "organization": "Alibaba", "configuration": "Published task-average score (0–100)", "score": 9.2, "raw_score": 9.2, "raw_scale": 1, "source_record": "Qwen-VL|Published task-average score (0–100)" }, { "id": "40de65ed37dbab95d945", "model": "LLaVA-Next-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 9.1, "raw_score": 9.1, "raw_scale": 1, "source_record": "LLaVA-Next-8B|Published task-average score (0–100)" }, { "id": "d89c603b41ae655d1b5c", "model": "Janus-1.3B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 8.6, "raw_score": 8.6, "raw_scale": 1, "source_record": "Janus-1.3B|Published task-average score (0–100)" }, { "id": "31d293a88edc988e32c8", "model": "ViLA1.5-8B", "organization": "", "configuration": "Published task-average score (0–100)", "score": 8.2, "raw_score": 8.2, "raw_scale": 1, "source_record": "ViLA1.5-8B|Published task-average score (0–100)" }, { "id": "909cbea6eee7e6f3c418", "model": "DocOwl2", "organization": "", "configuration": "Published task-average score (0–100)", "score": 7.3, "raw_score": 7.3, "raw_scale": 1, "source_record": "DocOwl2|Published task-average score (0–100)" }, { "id": "2d07c8a080894b999f41", "model": "CogVLM-chat", "organization": "", "configuration": "Published task-average score (0–100)", "score": 5.9, "raw_score": 5.9, "raw_scale": 1, "source_record": "CogVLM-chat|Published task-average score (0–100)" }, { "id": "5533541e15b96bd6546c", "model": "TextHarmony", "organization": "", "configuration": "Published task-average score (0–100)", "score": 5.6, "raw_score": 5.6, "raw_scale": 1, "source_record": "TextHarmony|Published task-average score (0–100)" }, { "id": "c08e1c1c46b906392831", "model": "UReader", "organization": "", "configuration": "Published task-average score (0–100)", "score": 5.5, "raw_score": 5.5, "raw_scale": 1, "source_record": "UReader|Published task-average score (0–100)" }, { "id": "432d9ac1f7fb97ab26f3", "model": "EMU2-chat", "organization": "", "configuration": "Published task-average score (0–100)", "score": 3.9, "raw_score": 3.9, "raw_scale": 1, "source_record": "EMU2-chat|Published task-average score (0–100)" }, { "id": "73d40ed0585a5ca446e1", "model": "LLaVAR", "organization": "", "configuration": "Published task-average score (0–100)", "score": 3.1, "raw_score": 3.1, "raw_scale": 1, "source_record": "LLaVAR|Published task-average score (0–100)" } ] }, { "id": "simpleqa-verified-epoch", "name": "SimpleQA Verified (Epoch)", "group": "知识与推理", "unit": "%", "metric": "Mean accuracy", "protocol": { "en": "Epoch primary evaluations using the anti-abstention prompt introduced on 2026-08-27. Earlier prompt runs and best-of-scorer scores are excluded.", "zh": "Epoch 直接评测，使用 2026-08-27 起的防弃答提示；不混入旧提示或挑选最佳评分器的成绩。" }, "source": { "data_url": "https://epoch.ai/data/benchmark_data.zip", "sha256": "c4683724f4e6c76fef1b5fedb63726938cfdde1a241eba20fec3e7c3896dcde4", "retrieved_at": "2026-11-01T16:38:03.969694+00:00", "name": "Epoch AI", "url": "https://epoch.ai/benchmarks/simple-qa-verified", "selector": "simpleqa_verified.csv: mean_score, Started at >= 2026-08-27; latest model/configuration", "status": "current_snapshot", "kind": "evaluator", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" } }, "results": [ { "id": "76200e7ea527c1b8754d", "model": "gemini-4-argon_high", "organization": "Google DeepMind", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.76, "raw_score": 0.76, "raw_scale": 1, "source_record": "iq4Y9tXpiyLP25yotySxJW", "evaluated_at": "2026-10-13T17:15:14.000Z" }, { "id": "4d79aec47f381857f71a", "model": "gpt-6-astra_max", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.756, "raw_score": 0.756, "raw_scale": 1, "source_record": "PppwnX7EjXx2SeTcFN8Ujn", "evaluated_at": "2026-08-30T14:57:43.000Z" }, { "id": "c6535cee6149066b5d33", "model": "gpt-6.1-sol_max", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.739, "raw_score": 0.739, "raw_scale": 1, "source_record": "dfT5VD7cQug8XW6GQHyJUF", "evaluated_at": "2026-09-29T22:00:34.000Z" }, { "id": "96f071ce7ee70c4c00d1", "model": "claude-opus-5-5_max", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.722, "raw_score": 0.722, "raw_scale": 1, "source_record": "CgLfiGcNwMPVzxr8GdNUAK", "evaluated_at": "2026-09-22T18:27:39.000Z" }, { "id": "280354648d48446b979a", "model": "claude-fable-5-1_max", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.708, "raw_score": 0.708, "raw_scale": 1, "source_record": "ExyL3GN8s34tizrXV3VmS3", "evaluated_at": "2026-09-01T18:31:59.000Z" }, { "id": "73770d949ed4f0a1d827", "model": "gemini-3.8-flash_high", "organization": "Google DeepMind", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.697, "raw_score": 0.697, "raw_scale": 1, "source_record": "gZ4bwozTKtpmZTqnqX3a33", "evaluated_at": "2026-09-02T17:15:36.000Z" }, { "id": "b4dcfbaa7e014ecebb4e", "model": "gemini-3.7-flash_high", "organization": "Google DeepMind", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.692, "raw_score": 0.692, "raw_scale": 1, "source_record": "84ARuGyhQsjgh8umRmBvas", "evaluated_at": "2026-08-27T19:30:25.000Z" }, { "id": "c3797ccbf95a168b9032", "model": "gemini-3-flash-preview_high", "organization": "Google DeepMind", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.668, "raw_score": 0.668, "raw_scale": 1, "source_record": "7cc6cDCXjsPjtKt36bbeMR", "evaluated_at": "2026-08-27T19:30:06.000Z" }, { "id": "86f18a14ef7d000b442c", "model": "gemini-3.5-flash_high", "organization": "Google DeepMind", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.662, "raw_score": 0.662, "raw_scale": 1, "source_record": "GJ6YsSuzyRD9YXe6SBwvsu", "evaluated_at": "2026-08-27T19:30:35.000Z" }, { "id": "9c314838687730608050", "model": "gemini-3.6-flash_high", "organization": "Google DeepMind", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.662, "raw_score": 0.662, "raw_scale": 1, "source_record": "F83pa7Gfsb4UNVttTnS5Hk", "evaluated_at": "2026-08-27T19:30:09.000Z" }, { "id": "5a949a1ee33ef9a82190", "model": "gpt-5.5_xhigh", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.63, "raw_score": 0.63, "raw_scale": 1, "source_record": "TijCafTTJUFrJ6jPGvBbgq", "evaluated_at": "2026-08-27T19:28:25.000Z" }, { "id": "97673604408bc400097e", "model": "gpt-6-sol_max", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.607, "raw_score": 0.607, "raw_scale": 1, "source_record": "donMeEKBZZdqpQsojKWJEm", "evaluated_at": "2026-09-22T19:59:24.000Z" }, { "id": "f08dd4c5ed69e6e4f602", "model": "muse-spark-1.2_xhigh", "organization": "Meta AI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.6030150753768844, "raw_score": 0.6030150753768844, "raw_scale": 1, "source_record": "mLLmVVvo6t23MM5yxq7ryW", "evaluated_at": "2026-08-27T19:36:49.000Z" }, { "id": "a6ac55cf5748f6ef5196", "model": "muse-spark-1.1", "organization": "Meta AI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.5778894472361809, "raw_score": 0.5778894472361809, "raw_scale": 1, "source_record": "bCdwpRXgZtvQLYC4wA9JXr", "evaluated_at": "2026-08-31T19:21:58.000Z" }, { "id": "e278cab2979a17ac07f7", "model": "gpt-6.1-terra_max", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.573, "raw_score": 0.573, "raw_scale": 1, "source_record": "Bixeio8A6SejZtUzFXFSXK", "evaluated_at": "2026-10-26T20:00:37.000Z" }, { "id": "5d87be281908c5e92dd3", "model": "Muse-Spark-Open-816B-A42B_xhigh", "organization": "Meta AI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.566, "raw_score": 0.566, "raw_scale": 1, "source_record": "dNuxLdV4DWqfSvtTq6NnK8", "evaluated_at": "2026-10-15T18:52:04.000Z" }, { "id": "25e98f19d2dbd13a3f93", "model": "grok-4.7_xhigh", "organization": "xAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.56, "raw_score": 0.56, "raw_scale": 1, "source_record": "6izDCw5ygCLZJEKPS8jV5T", "evaluated_at": "2026-09-22T21:49:22.000Z" }, { "id": "459a2fff392f9b08c637", "model": "qwen3.7-max", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.5576730190571715, "raw_score": 0.5576730190571715, "raw_scale": 1, "source_record": "DMjUrG6eZEqwcQ4SqUoViu", "evaluated_at": "2026-08-27T19:34:09.000Z" }, { "id": "e87214460d719603e987", "model": "claude-opus-4-8_max", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.53, "raw_score": 0.53, "raw_scale": 1, "source_record": "ViC6pxQZUVS3E9M9iEubF7", "evaluated_at": "2026-08-27T19:27:24.000Z" }, { "id": "f2bc5706da9bde588b30", "model": "deepseek-v4-pro-0813_max", "organization": "DeepSeek", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.5290581162324649, "raw_score": 0.5290581162324649, "raw_scale": 1, "source_record": "2xfFf6KZAp2gUWYhHGkFgs", "evaluated_at": "2026-08-27T19:30:20.000Z" }, { "id": "874d8d0171d44d360d7a", "model": "qwen3.6-max-preview", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.5200400801603207, "raw_score": 0.5200400801603207, "raw_scale": 1, "source_record": "iyPiR8GUuyNNA6Fssziq32", "evaluated_at": "2026-08-27T19:33:58.000Z" }, { "id": "c35a2b0fb0ac820e7211", "model": "claude-opus-4-7_xhigh", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.517, "raw_score": 0.517, "raw_scale": 1, "source_record": "2c4rvmf5N7QXTugrFLMZ83", "evaluated_at": "2026-08-27T19:27:26.000Z" }, { "id": "bd5cc63009da2b44afa3", "model": "kimi-k3_max", "organization": "Moonshot", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.506, "raw_score": 0.506, "raw_scale": 1, "source_record": "mZLWt2JSYvJyGhMvji7CT8", "evaluated_at": "2026-08-27T19:36:12.000Z" }, { "id": "45fda460db41b0b35847", "model": "gpt-5-2025-08-07_high", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.501, "raw_score": 0.501, "raw_scale": 1, "source_record": "gTLurbyKU63i2opsGkJGwh", "evaluated_at": "2026-08-27T19:28:49.000Z" }, { "id": "98b647376af4123fbb5e", "model": "o3-2025-04-16_high", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.494, "raw_score": 0.494, "raw_scale": 1, "source_record": "CxHSa2gY7ySiPPGTZEZaRt", "evaluated_at": "2026-08-27T19:28:51.000Z" }, { "id": "37f9a8f352f39e1830b7", "model": "grok-4.6_high", "organization": "xAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.493, "raw_score": 0.493, "raw_scale": 1, "source_record": "TpHYtuPG6kYZFvMK7Z6Hqj", "evaluated_at": "2026-08-27T19:29:53.000Z" }, { "id": "d77df68049726854731d", "model": "grok-4.6_xhigh", "organization": "xAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.489, "raw_score": 0.489, "raw_scale": 1, "source_record": "dKCWFaoGCWJTnD5CPWQGru", "evaluated_at": "2026-08-27T19:29:19.000Z" }, { "id": "d9823a88453e8d9f3b60", "model": "qwen3-max-2025-09-23", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.4874874874874875, "raw_score": 0.4874874874874875, "raw_scale": 1, "source_record": "fXG2GPt5F25tHRTPhAoZ4K", "evaluated_at": "2026-08-27T19:32:02.000Z" }, { "id": "65f54574a99cb7dfcfd2", "model": "grok-4.5_high", "organization": "xAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.483, "raw_score": 0.483, "raw_scale": 1, "source_record": "7HWW3sjRstKzb9JpcnvLaV", "evaluated_at": "2026-08-27T19:29:30.000Z" }, { "id": "5dc307e8268a1a468f2c", "model": "gpt-5.1-2025-11-13_high", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.48, "raw_score": 0.48, "raw_scale": 1, "source_record": "QCLnRy69NBH9ip3VYtQneh", "evaluated_at": "2026-08-27T19:28:24.000Z" }, { "id": "e09fe2f3d02505e8ef30", "model": "qwen3.8-max-0902_xhigh", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.473, "raw_score": 0.473, "raw_scale": 1, "source_record": "nBcLJJomVoeg2zh5F8Mnsu", "evaluated_at": "2026-09-02T18:04:58.000Z" }, { "id": "d21e619e3fb040365c70", "model": "claude-opus-4-6_max", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.47, "raw_score": 0.47, "raw_scale": 1, "source_record": "85fhMxLaebWtLoi6oLpTbi", "evaluated_at": "2026-08-27T19:27:45.000Z" }, { "id": "6ccc2352ea660add64af", "model": "deepseek-v4-pro_max", "organization": "DeepSeek", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.46993987975951906, "raw_score": 0.46993987975951906, "raw_scale": 1, "source_record": "Sq9xqkVTmFvELpQy9KH5TM", "evaluated_at": "2026-08-27T19:30:29.000Z" }, { "id": "9a6ba68664a06986670a", "model": "claude-sonnet-5-5_max", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.465, "raw_score": 0.465, "raw_scale": 1, "source_record": "GtJ5yd3ZPQ7HVcm6Q9ssQz", "evaluated_at": "2026-09-29T15:15:14.000Z" }, { "id": "fbd355fe566ecdde698a", "model": "gpt-5.4-pro-2026-03-05_xhigh", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.463, "raw_score": 0.463, "raw_scale": 1, "source_record": "RfSJbuuuMkeVHQj7xrth4u", "evaluated_at": "2026-08-27T19:28:24.000Z" }, { "id": "cab56ad3fca5d7327c4e", "model": "qwen3.8-max_xhigh", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.458, "raw_score": 0.458, "raw_scale": 1, "source_record": "bfftd5EzUK9yAekafEofvK", "evaluated_at": "2026-08-27T19:30:55.000Z" }, { "id": "cce96ceb847a69c99b1f", "model": "claude-opus-4-5-20251101_32K", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.457, "raw_score": 0.457, "raw_scale": 1, "source_record": "CA5LW4phRnHSsZUGXEm8H6", "evaluated_at": "2026-08-27T19:28:02.000Z" }, { "id": "0ef1c65e5d916234c4b3", "model": "gpt-5.4-2026-03-05_xhigh", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.451, "raw_score": 0.451, "raw_scale": 1, "source_record": "AzCkm282PLhDdLJ8DnAQQc", "evaluated_at": "2026-08-27T19:28:16.000Z" }, { "id": "78a90560c7806bfc5739", "model": "qwen3.6-plus", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.44144144144144143, "raw_score": 0.44144144144144143, "raw_scale": 1, "source_record": "4LWNorP8Fy8V8doxirNFcH", "evaluated_at": "2026-08-27T19:34:06.000Z" }, { "id": "3508680e09866d3cd279", "model": "deepseek-v4.1-flash_max", "organization": "DeepSeek", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.429, "raw_score": 0.429, "raw_scale": 1, "source_record": "3JJ4ChNxMo8CGAh3HrKNrw", "evaluated_at": "2026-10-13T12:59:33.000Z" }, { "id": "73ced3628b0d541bc75b", "model": "deepseek-v4.1-flash-1017_max", "organization": "DeepSeek", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.425, "raw_score": 0.425, "raw_scale": 1, "source_record": "Ei4XWaCH5drYUnDRo2MYCW", "evaluated_at": "2026-10-19T22:38:54.000Z" }, { "id": "7bdbe62df6efb5b28d75", "model": "gpt-6-luna_max", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.414, "raw_score": 0.414, "raw_scale": 1, "source_record": "HX9sRVWJtZiNvneALnVzs8", "evaluated_at": "2026-09-22T22:14:05.000Z" }, { "id": "80e425dc42e80a770a96", "model": "o1-2024-12-17_high", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.411, "raw_score": 0.411, "raw_scale": 1, "source_record": "LQfu4ngArcgMEyQZxuR7r4", "evaluated_at": "2026-08-31T22:01:34.000Z" }, { "id": "2fd60075da7e890750e0", "model": "glm-5.3_max", "organization": "Z.ai (Zhipu AI)", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.41, "raw_score": 0.41, "raw_scale": 1, "source_record": "TY8R8FV7zCwByEipkqyg2P", "evaluated_at": "2026-08-28T17:00:06.000Z" }, { "id": "e55a55898f5403385bc1", "model": "Qwen3-235B-A22B-Thinking-2507", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.4044044044044044, "raw_score": 0.4044044044044044, "raw_scale": 1, "source_record": "4ACEC4x26HWRGiAepHGw43", "evaluated_at": "2026-08-27T19:36:29.000Z" }, { "id": "3536187d7028ecfce952", "model": "Inkling_xhigh", "organization": "Thinking Machines", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.403, "raw_score": 0.403, "raw_scale": 1, "source_record": "bpnzC6CCi3hWuKCfqX4KEY", "evaluated_at": "2026-08-27T19:36:45.000Z" }, { "id": "a3b8dc78e38ede8f8fdc", "model": "gpt-5.2-2025-12-11_xhigh", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.371, "raw_score": 0.371, "raw_scale": 1, "source_record": "E3325ytkSNYP8ez53VA4fE", "evaluated_at": "2026-08-27T19:27:55.000Z" }, { "id": "d45c5bb918258b8694c1", "model": "kimi-k2.7-code", "organization": "Moonshot", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.365, "raw_score": 0.365, "raw_scale": 1, "source_record": "5AijYw5bFJtSuahnbzNo6Z", "evaluated_at": "2026-08-27T19:36:03.000Z" }, { "id": "33ee711ee355771bf268", "model": "gpt-5.2-2025-12-11_high", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.343, "raw_score": 0.343, "raw_scale": 1, "source_record": "Xp4qkZ32zjDeSYii2eQpnm", "evaluated_at": "2026-08-27T19:28:37.000Z" }, { "id": "e91f7442fb3c2a6c4e98", "model": "kimi-k2.5", "organization": "Moonshot", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.343, "raw_score": 0.343, "raw_scale": 1, "source_record": "RbXXg4JEbTSBnzw3CMSHnD", "evaluated_at": "2026-08-27T19:35:45.000Z" }, { "id": "301a3f95eaec576ceddc", "model": "glm-5.2_max", "organization": "Z.ai (Zhipu AI)", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.342, "raw_score": 0.342, "raw_scale": 1, "source_record": "WgjFZzojpiv54XqUhvfuUG", "evaluated_at": "2026-08-27T19:36:05.000Z" }, { "id": "a18cacc361b6c6d25f98", "model": "glm-5.1", "organization": "Z.ai (Zhipu AI)", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.34, "raw_score": 0.34, "raw_scale": 1, "source_record": "R2fMFAtRXah864hMifdQGV", "evaluated_at": "2026-08-27T19:36:21.000Z" }, { "id": "b2cde732c89f801fa443", "model": "claude-sonnet-5_max", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.337, "raw_score": 0.337, "raw_scale": 1, "source_record": "LPo74T9L3CawnKX5LLUrSZ", "evaluated_at": "2026-08-27T19:27:42.000Z" }, { "id": "324b109922316a0ec160", "model": "deepseek-v4-flash-0731_max", "organization": "DeepSeek", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.33633633633633636, "raw_score": 0.33633633633633636, "raw_scale": 1, "source_record": "TqFmiGvqozt7TSZYmhgF8m", "evaluated_at": "2026-08-27T19:30:16.000Z" }, { "id": "c15c789a998b045b168a", "model": "grok-4.3_high", "organization": "xAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.332, "raw_score": 0.332, "raw_scale": 1, "source_record": "Y39ZRp553ULpUDBV2XrBbc", "evaluated_at": "2026-08-27T19:29:33.000Z" }, { "id": "e657f6460e0fa61c7f6e", "model": "claude-sonnet-5_xhigh", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.329, "raw_score": 0.329, "raw_scale": 1, "source_record": "Qw4tGw38pk4sdsEehhrLSR", "evaluated_at": "2026-08-27T19:28:11.000Z" }, { "id": "6937d1445d3d6192b5a4", "model": "claude-sonnet-4-6_max", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.328, "raw_score": 0.328, "raw_scale": 1, "source_record": "Zs5fCvNzaz8LhqgwDSAztU", "evaluated_at": "2026-08-27T19:27:37.000Z" }, { "id": "def93fdd5754d11ec27d", "model": "gpt-5.2-2025-12-11_low", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.328, "raw_score": 0.328, "raw_scale": 1, "source_record": "Cyw3GA9a64AmT4pqLAhtTF", "evaluated_at": "2026-08-27T19:29:12.000Z" }, { "id": "5d952dcdb31e35614692", "model": "gpt-5.2-2025-12-11_medium", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.327, "raw_score": 0.327, "raw_scale": 1, "source_record": "esr5dcSuHiVoteo6vZfHjP", "evaluated_at": "2026-08-27T19:29:07.000Z" }, { "id": "55c7862504fff8328d4b", "model": "glm-4.7", "organization": "Z.ai (Zhipu AI)", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.322, "raw_score": 0.322, "raw_scale": 1, "source_record": "aYau6EG7CCaWVSyVq3vvQc", "evaluated_at": "2026-08-27T19:36:02.000Z" }, { "id": "7a6a787fbb7b183601af", "model": "gpt-4.1-2025-04-14", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.311, "raw_score": 0.311, "raw_scale": 1, "source_record": "avdYKfKUuY6EvfVdxftBE8", "evaluated_at": "2026-08-31T22:01:48.000Z" }, { "id": "84a5e44f7d8547eff847", "model": "claude-sonnet-4-5-20250929_59K", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.307, "raw_score": 0.307, "raw_scale": 1, "source_record": "XQEGUQjpXupjdyKoGuSmum", "evaluated_at": "2026-08-27T19:28:19.000Z" }, { "id": "368861cc5345b2972966", "model": "grok-4.20-0309-reasoning", "organization": "xAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.302, "raw_score": 0.302, "raw_scale": 1, "source_record": "GVm7khGvtdnQKGfkgAHD8q", "evaluated_at": "2026-08-27T19:29:44.000Z" }, { "id": "801bbf21894cc73c6015", "model": "gpt-5.4-mini-2026-03-17_high", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.294, "raw_score": 0.294, "raw_scale": 1, "source_record": "Z3pjKgAULY9j9wmzQpdqZG", "evaluated_at": "2026-08-27T19:28:49.000Z" }, { "id": "395b85dfded6959ed94e", "model": "gpt-4o-2024-08-06", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.26, "raw_score": 0.26, "raw_scale": 1, "source_record": "bDFqfWxMG7gLDq4Mz8PRTP", "evaluated_at": "2026-08-31T22:02:02.000Z" }, { "id": "b6eae5380a57b80a4c48", "model": "qwen3.5-plus", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.2535070140280561, "raw_score": 0.2535070140280561, "raw_scale": 1, "source_record": "jrawnM2wHFbVfJFJsap4kS", "evaluated_at": "2026-08-27T21:29:42.000Z" }, { "id": "0d77cd53ef8fc87f20b6", "model": "claude-haiku-5-5_max", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.238, "raw_score": 0.238, "raw_scale": 1, "source_record": "NwnrUMtveqHDYB2Um4ycmA", "evaluated_at": "2026-10-09T14:36:06.000Z" }, { "id": "d6456316c5f85c4e46b9", "model": "qwen3.5-flash", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.2032032032032032, "raw_score": 0.2032032032032032, "raw_scale": 1, "source_record": "K8zTckSXLKLesoLBv24TWc", "evaluated_at": "2026-08-27T19:32:07.000Z" }, { "id": "58dd1d30b0b1c3afa865", "model": "mistral-large-4-0_high", "organization": "Mistral AI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.2, "raw_score": 0.2, "raw_scale": 1, "source_record": "hvNNKdnzLvr9X8zVxWKcfk", "evaluated_at": "2026-10-09T14:12:09.000Z" }, { "id": "d2dc89c3c8933c013944", "model": "o4-mini-2025-04-16_low", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.196, "raw_score": 0.196, "raw_scale": 1, "source_record": "Dzky9brdTFbSoVd2zTKiUp", "evaluated_at": "2026-08-27T19:29:17.000Z" }, { "id": "aa902a3bc4cf27322777", "model": "inkling-small_xhigh", "organization": "Thinking Machines", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.191, "raw_score": 0.191, "raw_scale": 1, "source_record": "eJfbDKqeg9kNXFoxpeV7HG", "evaluated_at": "2026-08-27T19:36:17.000Z" }, { "id": "ebeeb9267b63697fcf12", "model": "o4-mini-2025-04-16_high", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.188, "raw_score": 0.188, "raw_scale": 1, "source_record": "gaGdPvvfvAw7MjP4HaVQCT", "evaluated_at": "2026-08-27T19:28:50.000Z" }, { "id": "370514e0abc0ef4fea64", "model": "qwen3.6-flash", "organization": "Alibaba", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.15915915915915915, "raw_score": 0.15915915915915915, "raw_scale": 1, "source_record": "FoMSS4FmH3VxpsvZiUJPCv", "evaluated_at": "2026-08-27T19:33:42.000Z" }, { "id": "5bd87e8854cb25d57e9e", "model": "o3-mini-2025-01-31_high", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.153, "raw_score": 0.153, "raw_scale": 1, "source_record": "UCba4H3NNtUiWXMisWnh9g", "evaluated_at": "2026-08-31T22:01:45.000Z" }, { "id": "f593811f76f8c542a664", "model": "gpt-4.1-mini-2025-04-14", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.127, "raw_score": 0.127, "raw_scale": 1, "source_record": "gYV4CJTjSd7VonfzMmnwcw", "evaluated_at": "2026-08-31T22:01:51.000Z" }, { "id": "1f93757a20bbd1db1ebe", "model": "claude-3-opus-20240229", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.126, "raw_score": 0.126, "raw_scale": 1, "source_record": "NxVp4VF8uQbL5S6hx6HFRB", "evaluated_at": "2026-08-31T18:18:52.000Z" }, { "id": "f64a1b3323ad6d09705a", "model": "claude-haiku-4-5-20251001_32K", "organization": "Anthropic", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.126, "raw_score": 0.126, "raw_scale": 1, "source_record": "VhBM9HgJ2RuSVwXriRXMVJ", "evaluated_at": "2026-08-27T19:27:58.000Z" }, { "id": "a0245b964371ad534e97", "model": "gpt-5.4-nano-2026-03-17_high", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.117, "raw_score": 0.117, "raw_scale": 1, "source_record": "3QBsxKkbF2R2LEXTd6HDrJ", "evaluated_at": "2026-08-27T19:28:50.000Z" }, { "id": "2e7ebbdc7f6e951ab8b8", "model": "gemma-4-31b-it", "organization": "Google DeepMind", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.104, "raw_score": 0.104, "raw_scale": 1, "source_record": "D5cbXacPnJ8akq2jAtifEQ", "evaluated_at": "2026-08-27T19:30:07.000Z" }, { "id": "0bddaf173a97badc385d", "model": "gpt-4o-mini-2024-07-18", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.083, "raw_score": 0.083, "raw_scale": 1, "source_record": "DLr586bZUHpghAf8m3GRSm", "evaluated_at": "2026-08-31T22:02:07.000Z" }, { "id": "26750069897e919e7c2a", "model": "gpt-4.1-nano-2025-04-14", "organization": "OpenAI", "configuration": "Epoch; anti-abstention prompt (2026-08-27 onward)", "score": 0.06, "raw_score": 0.06, "raw_scale": 1, "source_record": "A4JEeJR5LQfLR3Hb8z7kL2", "evaluated_at": "2026-08-31T22:01:58.000Z" } ] }, { "id": "livebench-2-20261025", "name": "LiveBench 2.0 (2026-10-25)", "group": "综合", "unit": "score", "metric": "Weighted overall", "protocol": { "en": "2026-10-25 release. Overall is reproduced from the official category weights: Long Agentic 2, Coding and Design 1.5, others 1. Composite score, not accuracy.", "zh": "2026-10-25 批次。按官方权重汇总：Long Agentic 为 2，Coding、Design 为 1.5，其余为 1。综合分数不等同于准确率。" }, "source": { "data_url": "https://livebench.ai/data/20261025/table.csv", "sha256": "a790e07b2ed710fbaa90eec799fcdfcbc04e4d37c16f7dc45cb3071717694d02", "retrieved_at": "2026-11-01T16:41:23.172406+00:00", "name": "LiveBench", "url": "https://livebench.ai/", "selector": "20261025/table.csv; category means weighted by categories.json _overall_weights", "status": "current_snapshot", "kind": "official_leaderboard", "artifacts": [ { "data_url": "https://livebench.ai/data/20261025/categories.json", "sha256": "b5366a69719445fa91e970de6f248c666115bbc5d8b8d134f33bb5fcc399a7ce", "retrieved_at": "2026-11-01T16:41:23.175441+00:00" }, { "data_url": "https://livebench.ai/static/js/main.1e1ce12d.js", "sha256": "b4158e5400886ba821e3c3b284a4a0b36687ffd04faedbe8a0492cf29d79ee8f", "retrieved_at": "2026-11-01T16:39:18.696956+00:00" } ] }, "results": [ { "id": "7f4847a2f798a5848591", "model": "Claude Fable 5.1", "organization": "Anthropic", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 85.69444444444444, "raw_score": 85.69444444444444, "raw_scale": 1, "source_record": "claude-fable-5.1", "category_scores": { "Long Agentic": 77.9, "Reasoning": 91.7, "Coding": 97.7, "Browser": 89.8, "Data Analysis": 80.3, "Research": 82.0, "Design": 83.4 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "576a599a64cb3dde05f1", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 85.63888888888889, "raw_score": 85.63888888888889, "raw_scale": 1, "source_record": "gpt-6-astra", "category_scores": { "Long Agentic": 77.5, "Reasoning": 92.7, "Coding": 93.7, "Browser": 91.3, "Data Analysis": 83.0, "Research": 85.2, "Design": 82.0 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "795584cf8178b32a9f18", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 85.14444444444445, "raw_score": 85.14444444444445, "raw_scale": 1, "source_record": "claude-opus-5.5", "category_scores": { "Long Agentic": 72.8, "Reasoning": 92.2, "Coding": 96.3, "Browser": 88.4, "Data Analysis": 80.3, "Research": 85.0, "Design": 86.9 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "dedfefd51120f28386c9", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 84.07222222222222, "raw_score": 84.07222222222222, "raw_scale": 1, "source_record": "gpt-6.1-sol", "category_scores": { "Long Agentic": 68.7, "Reasoning": 92.6, "Coding": 96.0, "Browser": 93.3, "Data Analysis": 82.7, "Research": 83.5, "Design": 82.1 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "4ae60805f94e4d2e6d47", "model": "Claude Sonnet 5.5", "organization": "Anthropic", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 80.03888888888889, "raw_score": 80.03888888888889, "raw_scale": 1, "source_record": "claude-sonnet-5.5", "category_scores": { "Long Agentic": 63.4, "Reasoning": 91.6, "Coding": 94.7, "Browser": 89.3, "Data Analysis": 59.5, "Research": 83.6, "Design": 85.0 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "fedcf0678c6ce4486fc3", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: high", "score": 79.30000000000001, "raw_score": 79.30000000000001, "raw_scale": 1, "source_record": "gemini-4-argon", "category_scores": { "Long Agentic": 61.2, "Reasoning": 95.1, "Coding": 89.7, "Browser": 82.8, "Data Analysis": 85.2, "Research": 83.1, "Design": 73.7 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "901b676976668ae9f873", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 77.78888888888889, "raw_score": 77.78888888888889, "raw_scale": 1, "source_record": "gpt-6.1-terra", "category_scores": { "Long Agentic": 48.6, "Reasoning": 88.7, "Coding": 94.3, "Browser": 93.1, "Data Analysis": 82.3, "Research": 73.9, "Design": 82.3 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "b8d1f5d038262fb52790", "model": "Muse Wonder 1.0", "organization": "Meta", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 74.15, "raw_score": 74.15, "raw_scale": 1, "source_record": "muse-wonder-1.0", "category_scores": { "Long Agentic": 52.6, "Reasoning": 90.6, "Coding": 86.2, "Browser": 60.0, "Data Analysis": 81.6, "Research": 88.0, "Design": 75.1 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "15bdae0c756666686aaf", "model": "DeepSeek V4.1 Flash 1017", "organization": "DeepSeek", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 72.86666666666666, "raw_score": 72.86666666666666, "raw_scale": 1, "source_record": "deepseek-v4.1-flash-1017", "category_scores": { "Long Agentic": 35.7, "Reasoning": 89.1, "Coding": 96.0, "Browser": 67.2, "Data Analysis": 79.5, "Research": 79.5, "Design": 83.4 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "f0bb7edc86af2251b77c", "model": "Kimi K3.1 Pro", "organization": "Moonshot AI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 71.28888888888889, "raw_score": 71.28888888888889, "raw_scale": 1, "source_record": "kimi-k3.1-pro", "category_scores": { "Long Agentic": 39.1, "Reasoning": 91.1, "Coding": 81.8, "Browser": 75.9, "Data Analysis": 79.1, "Research": 77.9, "Design": 77.8 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "459415a966d88d3e30f7", "model": "Grok 4.7", "organization": "xAI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: xhigh", "score": 70.13888888888889, "raw_score": 70.13888888888889, "raw_scale": 1, "source_record": "grok-4.7", "category_scores": { "Long Agentic": 46.1, "Reasoning": 82.7, "Coding": 84.5, "Browser": 49.3, "Data Analysis": 76.9, "Research": 86.4, "Design": 78.0 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "4d43bb0a4907c4de39a5", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 69.85, "raw_score": 69.85, "raw_scale": 1, "source_record": "claude-haiku-5.5", "category_scores": { "Long Agentic": 40.0, "Reasoning": 82.4, "Coding": 91.4, "Browser": 69.5, "Data Analysis": 53.1, "Research": 81.9, "Design": 83.1 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "42f4aabe21fcf6e90a9d", "model": "MiMo V2.6 Pro", "organization": "Xiaomi", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: high", "score": 68.47777777777777, "raw_score": 68.47777777777777, "raw_scale": 1, "source_record": "mimo-v2.6-pro", "category_scores": { "Long Agentic": 35.2, "Reasoning": 79.6, "Coding": 81.5, "Browser": 77.3, "Data Analysis": 72.3, "Research": 76.7, "Design": 78.5 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "deda3dc02a1d1575f15d", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 67.78888888888889, "raw_score": 67.78888888888889, "raw_scale": 1, "source_record": "gpt-5.6-terra", "category_scores": { "Long Agentic": 33.1, "Reasoning": 90.6, "Coding": 86.2, "Browser": 70.3, "Data Analysis": 79.3, "Research": 60.4, "Design": 76.0 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "44968622fb48d054af0f", "model": "Grok 4.6", "organization": "xAI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: xhigh", "score": 67.75, "raw_score": 67.75, "raw_scale": 1, "source_record": "grok-4.6", "category_scores": { "Long Agentic": 28.0, "Reasoning": 91.2, "Coding": 89.1, "Browser": 49.4, "Data Analysis": 75.3, "Research": 84.5, "Design": 79.8 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "b178a701b44eb1f58657", "model": "MiniMax M3.1 Flash", "organization": "MiniMax", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 66.89444444444445, "raw_score": 66.89444444444445, "raw_scale": 1, "source_record": "minimax-m3.1-flash", "category_scores": { "Long Agentic": 28.8, "Reasoning": 82.9, "Coding": 83.9, "Browser": 80.6, "Data Analysis": 71.4, "Research": 72.7, "Design": 74.0 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "9a7e60cd1c7bb9d54938", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 64.58888888888889, "raw_score": 64.58888888888889, "raw_scale": 1, "source_record": "gpt-6-luna", "category_scores": { "Long Agentic": 30.9, "Reasoning": 81.8, "Coding": 84.5, "Browser": 70.3, "Data Analysis": 73.4, "Research": 43.5, "Design": 82.5 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "80a8bf700c991501c5d0", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 64.57222222222222, "raw_score": 64.57222222222222, "raw_scale": 1, "source_record": "kimi-k3", "category_scores": { "Long Agentic": 26.4, "Reasoning": 90.7, "Coding": 59.3, "Browser": 75.2, "Data Analysis": 78.7, "Research": 79.0, "Design": 77.2 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "7c4537c17a16079a1fdc", "model": "Mistral Large 4 1029", "organization": "Mistral AI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: high", "score": 63.49444444444445, "raw_score": 63.49444444444445, "raw_scale": 1, "source_record": "mistral-large-4-0", "category_scores": { "Long Agentic": 27.5, "Reasoning": 85.2, "Coding": 95.4, "Browser": 17.4, "Data Analysis": 76.6, "Research": 79.7, "Design": 76.3 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "b3eba54f7f5df1a66df9", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 63.03333333333333, "raw_score": 63.03333333333333, "raw_scale": 1, "source_record": "muse-spark-1.3", "category_scores": { "Long Agentic": 25.5, "Reasoning": 89.7, "Coding": 90.2, "Browser": 30.9, "Data Analysis": 79.6, "Research": 68.6, "Design": 74.8 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "0d8724178b098d01df9c", "model": "GLM-5.3 Flash", "organization": "Z.AI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 62.06666666666667, "raw_score": 62.06666666666667, "raw_scale": 1, "source_record": "glm-5.3-flash", "category_scores": { "Long Agentic": 19.0, "Reasoning": 77.6, "Coding": 80.6, "Browser": 59.8, "Data Analysis": 76.4, "Research": 70.1, "Design": 77.2 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "bc33b81a67813da0dfd8", "model": "Qwen3.8 Max", "organization": "Alibaba", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 61.5611111111111, "raw_score": 61.5611111111111, "raw_scale": 1, "source_record": "qwen3.8-max", "category_scores": { "Long Agentic": 29.9, "Reasoning": 88.2, "Coding": 60.3, "Browser": 49.4, "Data Analysis": 78.4, "Research": 69.9, "Design": 78.6 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "319034f3e25d0d9bd6b3", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: high", "score": 59.205555555555556, "raw_score": 59.205555555555556, "raw_scale": 1, "source_record": "gemini-3.8-flash", "category_scores": { "Long Agentic": 16.5, "Reasoning": 89.3, "Coding": 82.2, "Browser": 45.1, "Data Analysis": 54.0, "Research": 79.1, "Design": 72.7 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "1f2eef3b56f3da2ec942", "model": "Step 5 (Preview)", "organization": "StepFun", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 58.60555555555555, "raw_score": 58.60555555555555, "raw_scale": 1, "source_record": "step-5-preview", "category_scores": { "Long Agentic": 19.4, "Reasoning": 83.8, "Coding": 85.6, "Browser": 24.8, "Data Analysis": 63.5, "Research": 72.8, "Design": 76.9 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "66fc7c05143b6b14bc5f", "model": "Qwen3.8 27B", "organization": "Alibaba", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 58.38333333333334, "raw_score": 58.38333333333334, "raw_scale": 1, "source_record": "qwen3.8-27b", "category_scores": { "Long Agentic": 20.6, "Reasoning": 80.0, "Coding": 86.8, "Browser": 24.2, "Data Analysis": 76.6, "Research": 59.4, "Design": 75.9 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "82c0181546fa95bb637d", "model": "Muse Spark Open", "organization": "Meta", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 51.62777777777777, "raw_score": 51.62777777777777, "raw_scale": 1, "source_record": "muse-spark-open", "category_scores": { "Long Agentic": 11.7, "Reasoning": 73.7, "Coding": 73.6, "Browser": 26.2, "Data Analysis": 60.2, "Research": 80.3, "Design": 60.3 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "894292a848c3f7ee05eb", "model": "Smaug-Nemotron-3.5-NVFP4", "organization": "Abacus.AI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 50.30555555555556, "raw_score": 50.30555555555556, "raw_scale": 1, "source_record": "smaug-nemotron-3.5-nvfp4", "category_scores": { "Long Agentic": 8.4, "Reasoning": 75.1, "Coding": 72.2, "Browser": 25.6, "Data Analysis": 65.1, "Research": 67.8, "Design": 62.7 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "36d738d59a9477ac6782", "model": "Nemotron 3.5 Super VL", "organization": "NVIDIA", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 50.227777777777774, "raw_score": 50.227777777777774, "raw_scale": 1, "source_record": "nemotron-super-vl-3.5", "category_scores": { "Long Agentic": 11.7, "Reasoning": 74.6, "Coding": 71.3, "Browser": 21.6, "Data Analysis": 64.6, "Research": 66.7, "Design": 62.8 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "538c89ff59cdcfdf7e9f", "model": "Inkling", "organization": "Thinking Machines", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 47.05, "raw_score": 47.05, "raw_scale": 1, "source_record": "inkling", "category_scores": { "Long Agentic": 8.5, "Reasoning": 78.3, "Coding": 64.0, "Browser": 12.0, "Data Analysis": 72.8, "Research": 59.9, "Design": 58.3 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "41b0650a9e79293e42c9", "model": "Nemotron 3.5 Super VL (NVFP4)", "organization": "NVIDIA", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 46.5, "raw_score": 46.5, "raw_scale": 1, "source_record": "nemotron-super-vl-3.5-nvfp4", "category_scores": { "Long Agentic": 7.3, "Reasoning": 71.7, "Coding": 61.3, "Browser": 18.0, "Data Analysis": 65.7, "Research": 70.9, "Design": 57.1 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "1382198253aa94ae5c88", "model": "Mistral Large 4", "organization": "Mistral AI", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: high", "score": 43.900000000000006, "raw_score": 43.900000000000006, "raw_scale": 1, "source_record": "mistral-large-4", "category_scores": { "Long Agentic": 8.8, "Reasoning": 83.9, "Coding": 60.0, "Browser": 0.6, "Data Analysis": 76.5, "Research": 28.1, "Design": 65.6 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } }, { "id": "f3e5e631eb3ee132e084", "model": "Gemma 4 31B", "organization": "Google", "configuration": "LiveBench 2.0; 2026-10-25; official weighted overall; reasoning effort: max", "score": 38.72222222222222, "raw_score": 38.72222222222222, "raw_scale": 1, "source_record": "gemma-4-31b", "category_scores": { "Long Agentic": 10.7, "Reasoning": 68.3, "Coding": 49.0, "Browser": 0.0, "Data Analysis": 53.3, "Research": 44.4, "Design": 58.4 }, "category_weights": { "Long Agentic": 2.0, "Reasoning": 1, "Coding": 1.5, "Browser": 1, "Data Analysis": 1, "Research": 1, "Design": 1.5 } } ] }, { "id": "ifeval-opencompass", "name": "IFEval (OpenCompass)", "group": "指令与长文", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Current OpenCompass academic release (2026-03-09); strict prompt-level accuracy.", "zh": "OpenCompass 官方直接评测。2026-03-09 批次，严格提示级准确率。" }, "source": { "data_url": "https://cdn.opencompass.org.cn/assets/research-rank/research-data.REALTIME.20260309.json", "sha256": "dc31e3998c2315b92b653507cfff525ff0c20c91df6f96a9892ee7cc2e145892", "retrieved_at": "2026-11-01T16:45:32.913736+00:00", "name": "OpenCompass", "url": "https://rank.opencompass.org.cn/leaderboard-llm-academic", "selector": "OverallTable.IFEval", "status": "current_snapshot", "kind": "evaluator", "artifacts": [ { "data_url": "https://rank.opencompass.org.cn/gw/opencompass-be/api/v1/rank/listRankTableAvailableMonths", "sha256": "613aac662142782331ef0cda890ad1ecb1311bcf20aca7fb4a2b3fb673ff38b4", "retrieved_at": "2026-11-01T16:45:17.980490+00:00" } ] }, "results": [ { "id": "33ba0d3910b4549525af", "model": "GPT-5-2025-08-07 (high)", "organization": "OpenAI", "configuration": "OpenCompass; Chat", "score": 0.946, "raw_score": 94.6, "raw_scale": 100, "source_record": "GPT-5-2025-08-07 (high)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "16dfc89e3b5ffb383fd8", "model": "JT-Think", "organization": "China Mobile", "configuration": "OpenCompass; Chat", "score": 0.946, "raw_score": 94.6, "raw_scale": 100, "source_record": "JT-Think|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "d7b1d1c41aab1ec9ddd4", "model": "TeleChat-thinking", "organization": "TeleAI", "configuration": "OpenCompass; Chat", "score": 0.943, "raw_score": 94.3, "raw_scale": 100, "source_record": "TeleChat-thinking|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "5070132bfde14252741e", "model": "Kimi-K2.5", "organization": "Moonshot", "configuration": "OpenCompass; Chat", "score": 0.9390000000000001, "raw_score": 93.9, "raw_scale": 100, "source_record": "Kimi-K2.5|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "36362bfd13f9b77d121a", "model": "GLM-5", "organization": "Zhipu AI", "configuration": "OpenCompass; Chat", "score": 0.932, "raw_score": 93.2, "raw_scale": 100, "source_record": "GLM-5|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "9982b6c896251ec78b8e", "model": "Step-3.5-Flash", "organization": "StepFun", "configuration": "OpenCompass; Chat", "score": 0.932, "raw_score": 93.2, "raw_scale": 100, "source_record": "Step-3.5-Flash|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "a6392f4fef306ab5fa95", "model": "o4-mini-2025-04-16 (high)", "organization": "OpenAI", "configuration": "OpenCompass; Chat", "score": 0.93, "raw_score": 93.0, "raw_scale": 100, "source_record": "o4-mini-2025-04-16 (high)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "e82e4c7d2401a709a34e", "model": "GPT-5-mini-2025-08-07 (high)", "organization": "OpenAI", "configuration": "OpenCompass; Chat", "score": 0.9279999999999999, "raw_score": 92.8, "raw_scale": 100, "source_record": "GPT-5-mini-2025-08-07 (high)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "40b58d46c3dc9db7bf94", "model": "Gemini-3-Pro-Preview", "organization": "Google", "configuration": "OpenCompass; Chat", "score": 0.9279999999999999, "raw_score": 92.8, "raw_scale": 100, "source_record": "Gemini-3-Pro-Preview|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "4e34ad3b8b1764904cbf", "model": "Kimi-K2-Thinking", "organization": "Moonshot", "configuration": "OpenCompass; Chat", "score": 0.924, "raw_score": 92.4, "raw_scale": 100, "source_record": "Kimi-K2-Thinking|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "390a2115ea406b1b0579", "model": "o3-2025-04-16", "organization": "OpenAI", "configuration": "OpenCompass; Chat", "score": 0.922, "raw_score": 92.2, "raw_scale": 100, "source_record": "o3-2025-04-16|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "2e70bdf54030d48f2452", "model": "DeepSeek-V3.2-Speciale", "organization": "DeepSeek", "configuration": "OpenCompass; Chat", "score": 0.917, "raw_score": 91.7, "raw_scale": 100, "source_record": "DeepSeek-V3.2-Speciale|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "fe7adc3e6ec6f7834429", "model": "Qwen3.5-397B-A17B", "organization": "Qwen", "configuration": "OpenCompass; Chat", "score": 0.915, "raw_score": 91.5, "raw_scale": 100, "source_record": "Qwen3.5-397B-A17B|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "43958a92306614a397df", "model": "MiniMax-M2.5", "organization": "MiniMax", "configuration": "OpenCompass; Chat", "score": 0.9109999999999999, "raw_score": 91.1, "raw_scale": 100, "source_record": "MiniMax-M2.5|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "b82884776dd11d506efe", "model": "GLM-4.7", "organization": "Zhipu AI", "configuration": "OpenCompass; Chat", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "GLM-4.7|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "3e94682a157c9ec1472c", "model": "GPT-5-nano-2025-08-07 (high)", "organization": "OpenAI", "configuration": "OpenCompass; Chat", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "GPT-5-nano-2025-08-07 (high)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "306f1ffecd7c212eea81", "model": "LongCat-Flash-Chat", "organization": "Meituan", "configuration": "OpenCompass; Chat", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "LongCat-Flash-Chat|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "0b9acb0661a23f9389b3", "model": "MiniMax-M2", "organization": "MiniMax", "configuration": "OpenCompass; Chat", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "MiniMax-M2|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "b66c6fada5045aa4468c", "model": "gpt-oss-120b (high)", "organization": "OpenAI", "configuration": "OpenCompass; Chat", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "gpt-oss-120b (high)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "c3541d0e83afaa45c98e", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "OpenCompass; Chat", "score": 0.9, "raw_score": 90.0, "raw_scale": 100, "source_record": "Gemini-2.5-Pro|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "7f328e601e8dfbb60721", "model": "DeepSeek-V3.2", "organization": "DeepSeek", "configuration": "OpenCompass; Chat", "score": 0.897, "raw_score": 89.7, "raw_scale": 100, "source_record": "DeepSeek-V3.2|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "1a77863934cfe942d703", "model": "Qwen3-30B-A3B-Thinking-2507", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.897, "raw_score": 89.7, "raw_scale": 100, "source_record": "Qwen3-30B-A3B-Thinking-2507|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "bc2c1aa372861d098136", "model": "MiMo-V2-Flash", "organization": "Xiaomi", "configuration": "OpenCompass; Chat", "score": 0.895, "raw_score": 89.5, "raw_scale": 100, "source_record": "MiMo-V2-Flash|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "23abde83673a83382433", "model": "Qwen3-Next-80B-A3B-Thinking", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.895, "raw_score": 89.5, "raw_scale": 100, "source_record": "Qwen3-Next-80B-A3B-Thinking|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "defd48d21cec5121f95d", "model": "ERNIE-4.5-300B-A47B-PT", "organization": "Baidu", "configuration": "OpenCompass; Chat", "score": 0.889, "raw_score": 88.9, "raw_scale": 100, "source_record": "ERNIE-4.5-300B-A47B-PT|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "2ed201250b22cfd3b7b0", "model": "gpt-oss-20b (high)", "organization": "OpenAI", "configuration": "OpenCompass; Chat", "score": 0.889, "raw_score": 88.9, "raw_scale": 100, "source_record": "gpt-oss-20b (high)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "98ac0162e13f6b63879c", "model": "Finix-P1-32B (Thinking)", "organization": "Ant Group", "configuration": "OpenCompass; Chat", "score": 0.887, "raw_score": 88.7, "raw_scale": 100, "source_record": "Finix-P1-32B (Thinking)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "65b3d0c93b1edcb8e81c", "model": "GLM-4.6", "organization": "Zhipu AI", "configuration": "OpenCompass; Chat", "score": 0.887, "raw_score": 88.7, "raw_scale": 100, "source_record": "GLM-4.6|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "7b79607d3257b23e35b8", "model": "Kimi-K2-Instruct", "organization": "Moonshot", "configuration": "OpenCompass; Chat", "score": 0.885, "raw_score": 88.5, "raw_scale": 100, "source_record": "Kimi-K2-Instruct|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "42757b2a8a47b4f61717", "model": "Qwen3-4B-Thinking-2507", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.885, "raw_score": 88.5, "raw_scale": 100, "source_record": "Qwen3-4B-Thinking-2507|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "7969af5626182dee8941", "model": "Claude Sonnet 4 (Thinking)", "organization": "Anthropic", "configuration": "OpenCompass; Chat", "score": 0.883, "raw_score": 88.3, "raw_scale": 100, "source_record": "Claude Sonnet 4 (Thinking)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "7e0b79e9e895d7a7b52b", "model": "Qwen3-235B-A22B-Instruct-2507", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.883, "raw_score": 88.3, "raw_scale": 100, "source_record": "Qwen3-235B-A22B-Instruct-2507|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "69f876b630556c830b56", "model": "GPT-4.1-20250414", "organization": "OpenAI", "configuration": "OpenCompass; Chat", "score": 0.882, "raw_score": 88.2, "raw_scale": 100, "source_record": "GPT-4.1-20250414|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "e98826bdc849dbc4413b", "model": "Qwen3-235B-A22B-Thinking-2507", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.878, "raw_score": 87.8, "raw_scale": 100, "source_record": "Qwen3-235B-A22B-Thinking-2507|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "1b0ef414d6f18f13c9e4", "model": "Qwen3-Next-80B-A3B-Instruct", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.8759999999999999, "raw_score": 87.6, "raw_scale": 100, "source_record": "Qwen3-Next-80B-A3B-Instruct|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "53a9d6c183bbffd7012d", "model": "Llama4-Maverick-17B-128E-Instruct", "organization": "Meta", "configuration": "OpenCompass; Chat", "score": 0.871, "raw_score": 87.1, "raw_scale": 100, "source_record": "Llama4-Maverick-17B-128E-Instruct|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "04e41d7d57391830a2af", "model": "Qwen3-32B (Thinking)", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.86, "raw_score": 86.0, "raw_scale": 100, "source_record": "Qwen3-32B (Thinking)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "270a4fb0d85dd106b15f", "model": "Qwen3-8B (Thinking)", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.856, "raw_score": 85.6, "raw_scale": 100, "source_record": "Qwen3-8B (Thinking)|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "bd5edf2dca5064b5debf", "model": "GLM-4.5", "organization": "Zhipu AI", "configuration": "OpenCompass; Chat", "score": 0.8540000000000001, "raw_score": 85.4, "raw_scale": 100, "source_record": "GLM-4.5|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "db599fcb1653bc4441b5", "model": "Qwen3-30B-A3B-Instruct-2507", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.8390000000000001, "raw_score": 83.9, "raw_scale": 100, "source_record": "Qwen3-30B-A3B-Instruct-2507|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "4190dbd045592e809501", "model": "Seed-OSS-36B-Instruct", "organization": "ByteDance", "configuration": "OpenCompass; Chat", "score": 0.8370000000000001, "raw_score": 83.7, "raw_scale": 100, "source_record": "Seed-OSS-36B-Instruct|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "423ff55fdcaa0a8ce256", "model": "Qwen3-4B-Instruct-2507", "organization": "Alibaba", "configuration": "OpenCompass; Chat", "score": 0.8240000000000001, "raw_score": 82.4, "raw_scale": 100, "source_record": "Qwen3-4B-Instruct-2507|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "d5aa36e432cf4f0eb90e", "model": "ERNIE-4.5-21B-A3B-Thinking", "organization": "Baidu", "configuration": "OpenCompass; Chat", "score": 0.812, "raw_score": 81.2, "raw_scale": 100, "source_record": "ERNIE-4.5-21B-A3B-Thinking|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "5e05f3025897fb268f94", "model": "Gemma-3-27B-it", "organization": "Google", "configuration": "OpenCompass; Chat", "score": 0.81, "raw_score": 81.0, "raw_scale": 100, "source_record": "Gemma-3-27B-it|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "06bc286018778b58ca66", "model": "DeepSeek-R1-0528", "organization": "DeepSeek", "configuration": "OpenCompass; Chat", "score": 0.8, "raw_score": 80.0, "raw_scale": 100, "source_record": "DeepSeek-R1-0528|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "bcc8f5e88329595d6947", "model": "doubao-seed-1.6-thinking-250615", "organization": "ByteDance", "configuration": "OpenCompass; Chat", "score": 0.789, "raw_score": 78.9, "raw_scale": 100, "source_record": "doubao-seed-1.6-thinking-250615|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "40b2a54d57edce3a8ec0", "model": "MiniCPM-SALA", "organization": "OpenBMB", "configuration": "OpenCompass; Chat", "score": 0.769, "raw_score": 76.9, "raw_scale": 100, "source_record": "MiniCPM-SALA|OpenCompass; Chat", "source_date": "2026/3/9" }, { "id": "1ba45b9f2b723c21c9a3", "model": "Hunyuan-A13B-Instruct", "organization": "Tencent", "configuration": "OpenCompass; Chat", "score": 0.603, "raw_score": 60.3, "raw_scale": 100, "source_record": "Hunyuan-A13B-Instruct|OpenCompass; Chat", "source_date": "2026/3/9" } ] }, { "id": "cmmlu-5shot", "name": "CMMLU (5-shot)", "group": "知识与推理", "unit": "%", "metric": "Average accuracy", "protocol": { "en": "Author-maintained README, 5-shot average across subjects. Latest published entries are retained; this table is not a current-model census.", "zh": "作者维护的 README，使用 5-shot 各科平均分；保留目前公开记录，该表不代表当前所有模型。" }, "source": { "data_url": "https://raw.githubusercontent.com/haonan-li/CMMLU/master/README.md", "sha256": "958e0b90d4258bb477b6f432d8eeb919148339efa8869f83e71a60d36ef3e24c", "retrieved_at": "2026-11-01T16:38:08.446804+00:00", "name": "CMMLU authors", "url": "https://github.com/haonan-li/CMMLU", "selector": "README Five-shot section: average column; exclude Random; no zero-shot mixing", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "ca1a16e96aa01e010a98", "model": "Lingzhi-72B-chat", "organization": "", "configuration": "5-shot; publisher README", "score": 0.9026000000000001, "raw_score": 90.26, "raw_scale": 100, "source_record": "Lingzhi-72B-chat|5-shot; publisher README" }, { "id": "52873dd1787815cdda54", "model": "Telechat2-35B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.9016, "raw_score": 90.16, "raw_scale": 100, "source_record": "Telechat2-35B|5-shot; publisher README" }, { "id": "cae1a157a23fc754f290", "model": "Spark 4.0-2024-10-14", "organization": "", "configuration": "5-shot; publisher README", "score": 0.9007, "raw_score": 90.07, "raw_scale": 100, "source_record": "Spark 4.0-2024-10-14|5-shot; publisher README" }, { "id": "8141910b759537a57c85", "model": "Qwen2-72B", "organization": "Alibaba", "configuration": "5-shot; publisher README", "score": 0.8965000000000001, "raw_score": 89.65, "raw_scale": 100, "source_record": "Qwen2-72B|5-shot; publisher README" }, { "id": "d44dcab7dcc91bbdce51", "model": "Jiutian-大模型", "organization": "", "configuration": "5-shot; publisher README", "score": 0.8859, "raw_score": 88.59, "raw_scale": 100, "source_record": "Jiutian-大模型|5-shot; publisher README" }, { "id": "40efded9c2affa0d3a86", "model": "Qwen1.5-110B", "organization": "Alibaba", "configuration": "5-shot; publisher README", "score": 0.8832, "raw_score": 88.32, "raw_scale": 100, "source_record": "Qwen1.5-110B|5-shot; publisher README" }, { "id": "82de5753f1fad4dcdf21", "model": "JIUTIAN-57B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.8739, "raw_score": 87.39, "raw_scale": 100, "source_record": "JIUTIAN-57B|5-shot; publisher README" }, { "id": "1a3e6fe9838d286e3040", "model": "Qwen2.5-72B", "organization": "Alibaba", "configuration": "5-shot; publisher README", "score": 0.8567, "raw_score": 85.67, "raw_scale": 100, "source_record": "Qwen2.5-72B|5-shot; publisher README" }, { "id": "c0bcfc35f5c9b8bc65f7", "model": "BlueLM", "organization": "", "configuration": "5-shot; publisher README", "score": 0.8559, "raw_score": 85.59, "raw_scale": 100, "source_record": "BlueLM|5-shot; publisher README" }, { "id": "030511e932bace29ac8f", "model": "Youyuanjian", "organization": "", "configuration": "5-shot; publisher README", "score": 0.8371999999999999, "raw_score": 83.72, "raw_scale": 100, "source_record": "Youyuanjian|5-shot; publisher README" }, { "id": "d0b0d1bcdbe379c165b0", "model": "Qwen1.5-72B", "organization": "Alibaba", "configuration": "5-shot; publisher README", "score": 0.8354, "raw_score": 83.54, "raw_scale": 100, "source_record": "Qwen1.5-72B|5-shot; publisher README" }, { "id": "89ca9a33fe7070ada997", "model": "Mind GPT", "organization": "", "configuration": "5-shot; publisher README", "score": 0.8284, "raw_score": 82.84, "raw_scale": 100, "source_record": "Mind GPT|5-shot; publisher README" }, { "id": "ef35f6f260c67c818cb8", "model": "PCI-TransGPT", "organization": "", "configuration": "5-shot; publisher README", "score": 0.8245999999999999, "raw_score": 82.46, "raw_scale": 100, "source_record": "PCI-TransGPT|5-shot; publisher README" }, { "id": "6fe9fc5f36239345b1f5", "model": "Qwen1.5-32B", "organization": "Alibaba", "configuration": "5-shot; publisher README", "score": 0.8225, "raw_score": 82.25, "raw_scale": 100, "source_record": "Qwen1.5-32B|5-shot; publisher README" }, { "id": "69e96aaa4c2f1abd0679", "model": "ZW-LM", "organization": "", "configuration": "5-shot; publisher README", "score": 0.8173, "raw_score": 81.73, "raw_scale": 100, "source_record": "ZW-LM|5-shot; publisher README" }, { "id": "d6975120dc6f70b3df5e", "model": "QuarkLLM", "organization": "", "configuration": "5-shot; publisher README", "score": 0.8027, "raw_score": 80.27, "raw_scale": 100, "source_record": "QuarkLLM|5-shot; publisher README" }, { "id": "7da2ea5393eb2827c6e9", "model": "ZhiLu-2-8B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.7995, "raw_score": 79.95, "raw_scale": 100, "source_record": "ZhiLu-2-8B|5-shot; publisher README" }, { "id": "57b0abbec823c1eda3cb", "model": "BlueLM-7B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.7426999999999999, "raw_score": 74.27, "raw_scale": 100, "source_record": "BlueLM-7B|5-shot; publisher README" }, { "id": "6e219527c9d31f83313c", "model": "Galaxy", "organization": "", "configuration": "5-shot; publisher README", "score": 0.7403, "raw_score": 74.03, "raw_scale": 100, "source_record": "Galaxy|5-shot; publisher README" }, { "id": "1359862d71f32f3f4c0a", "model": "Qwen1.5-7B", "organization": "Alibaba", "configuration": "5-shot; publisher README", "score": 0.725, "raw_score": 72.5, "raw_scale": 100, "source_record": "Qwen1.5-7B|5-shot; publisher README" }, { "id": "3761bb0927a5d17acb72", "model": "KwaiYii-66B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.7112, "raw_score": 71.12, "raw_scale": 100, "source_record": "KwaiYii-66B|5-shot; publisher README" }, { "id": "d4ee23ffa54813914478", "model": "XuanYuan-70B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.711, "raw_score": 71.1, "raw_scale": 100, "source_record": "XuanYuan-70B|5-shot; publisher README" }, { "id": "9711953174134fbb95c8", "model": "GPT4", "organization": "OpenAI", "configuration": "5-shot; publisher README", "score": 0.7095, "raw_score": 70.95, "raw_scale": 100, "source_record": "GPT4|5-shot; publisher README" }, { "id": "8c7dfd50023737bbd094", "model": "FanFan-1.5B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.665, "raw_score": 66.5, "raw_scale": 100, "source_record": "FanFan-1.5B|5-shot; publisher README" }, { "id": "57b488effd3e8d3fbe48", "model": "Mengzi-7B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.6640999999999999, "raw_score": 66.41, "raw_scale": 100, "source_record": "Mengzi-7B|5-shot; publisher README" }, { "id": "9d39d030e77907b3e117", "model": "Llama-3.1-70B-Instruct", "organization": "Meta", "configuration": "5-shot; publisher README", "score": 0.6437999999999999, "raw_score": 64.38, "raw_scale": 100, "source_record": "Llama-3.1-70B-Instruct|5-shot; publisher README" }, { "id": "78bfd9880b66911c1ad8", "model": "KwaiYii-13B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.6173, "raw_score": 61.73, "raw_scale": 100, "source_record": "KwaiYii-13B|5-shot; publisher README" }, { "id": "8226a16e9a187587500e", "model": "XuanYuan-13B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.6004999999999999, "raw_score": 60.05, "raw_scale": 100, "source_record": "XuanYuan-13B|5-shot; publisher README" }, { "id": "18661c421ba180098700", "model": "Qwen-7B", "organization": "Alibaba", "configuration": "5-shot; publisher README", "score": 0.5866, "raw_score": 58.66, "raw_scale": 100, "source_record": "Qwen-7B|5-shot; publisher README" }, { "id": "a85e64b56ba66a013908", "model": "MiLM-6B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.5717, "raw_score": 57.17, "raw_scale": 100, "source_record": "MiLM-6B|5-shot; publisher README" }, { "id": "3d5aec95b2be87c783c1", "model": "ZhiLu-13B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.5716, "raw_score": 57.16, "raw_scale": 100, "source_record": "ZhiLu-13B|5-shot; publisher README" }, { "id": "b223e405ec11aafd6e5c", "model": "Baichuan-13B", "organization": "Baichuan", "configuration": "5-shot; publisher README", "score": 0.5582, "raw_score": 55.82, "raw_scale": 100, "source_record": "Baichuan-13B|5-shot; publisher README" }, { "id": "9adc7dcd9f3536a9e17c", "model": "ChatGPT", "organization": "OpenAI", "configuration": "5-shot; publisher README", "score": 0.5550999999999999, "raw_score": 55.51, "raw_scale": 100, "source_record": "ChatGPT|5-shot; publisher README" }, { "id": "9c4d0f8d0113eaa87a58", "model": "ChatGLM2-6B", "organization": "Z AI", "configuration": "5-shot; publisher README", "score": 0.488, "raw_score": 48.8, "raw_scale": 100, "source_record": "ChatGLM2-6B|5-shot; publisher README" }, { "id": "a189f3ec9bfda63f0050", "model": "MiLM-1.3B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.4539, "raw_score": 45.39, "raw_scale": 100, "source_record": "MiLM-1.3B|5-shot; publisher README" }, { "id": "3d96c19ba1c1b7141c68", "model": "Baichuan-7B", "organization": "Baichuan", "configuration": "5-shot; publisher README", "score": 0.4443, "raw_score": 44.43, "raw_scale": 100, "source_record": "Baichuan-7B|5-shot; publisher README" }, { "id": "00f2ed61129376d135f2", "model": "Falcon-40B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.41450000000000004, "raw_score": 41.45, "raw_scale": 100, "source_record": "Falcon-40B|5-shot; publisher README" }, { "id": "0fa6d776da30cc6ca6f5", "model": "LLaMA-65B", "organization": "Meta", "configuration": "5-shot; publisher README", "score": 0.39799999999999996, "raw_score": 39.8, "raw_scale": 100, "source_record": "LLaMA-65B|5-shot; publisher README" }, { "id": "152cd001f34d9721207d", "model": "ChatGLM-6B", "organization": "Z AI", "configuration": "5-shot; publisher README", "score": 0.37479999999999997, "raw_score": 37.48, "raw_scale": 100, "source_record": "ChatGLM-6B|5-shot; publisher README" }, { "id": "2a99bc288eaf25c5421d", "model": "BatGPT-15B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.3716, "raw_score": 37.16, "raw_scale": 100, "source_record": "BatGPT-15B|5-shot; publisher README" }, { "id": "d579cc402c1588151e4a", "model": "BLOOMZ-7B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.3704, "raw_score": 37.04, "raw_scale": 100, "source_record": "BLOOMZ-7B|5-shot; publisher README" }, { "id": "7bb12f61d3fb412e0749", "model": "Llama-3-70B-Instruct", "organization": "Meta", "configuration": "5-shot; publisher README", "score": 0.3685, "raw_score": 36.85, "raw_scale": 100, "source_record": "Llama-3-70B-Instruct|5-shot; publisher README" }, { "id": "bf057a5e9c46278ac1d3", "model": "Chinese-LLaMA-13B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.32630000000000003, "raw_score": 32.63, "raw_scale": 100, "source_record": "Chinese-LLaMA-13B|5-shot; publisher README" }, { "id": "8d81b0ac4c54def35486", "model": "Bactrian-LLaMA-13B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.3188, "raw_score": 31.88, "raw_scale": 100, "source_record": "Bactrian-LLaMA-13B|5-shot; publisher README" }, { "id": "824391ac04bd44434d2a", "model": "MOSS-SFT-16B", "organization": "", "configuration": "5-shot; publisher README", "score": 0.2957, "raw_score": 29.57, "raw_scale": 100, "source_record": "MOSS-SFT-16B|5-shot; publisher README" } ] }, { "id": "gaia2-cli", "name": "Gaia2-CLI", "group": "Agent 与工具", "unit": "%", "metric": "pass@1", "protocol": { "en": "Current CLI track, OpenClaw 2026.4.1. Scores are reported by submitting teams. Not interchangeable with GAIA 2023 or the original Gaia2 track.", "zh": "当前 CLI 赛道，OpenClaw 2026.4.1。成绩由提交团队报告，不与 GAIA 2023 或原始 Gaia2 赛道混用。" }, "source": { "data_url": "https://meta-agents-research-environments-leaderboard.hf.space/config", "sha256": "17dc12c62adc965830d43e4798cb87040a693b48166c9420c1de6339a4af1a98", "retrieved_at": "2026-11-01T16:51:00.749860+00:00", "name": "Meta / Gaia2", "url": "https://huggingface.co/spaces/meta-agents-research-environments/leaderboard", "selector": "Gaia2-CLI HTML table; pass@1; not the older non-CLI dataframe", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "c583370397e17bd96f05", "model": "Claude Opus 4.6 (high)", "organization": "Anthropic", "configuration": "OpenClaw 2026.4.1", "score": 0.57, "raw_score": 57.0, "raw_scale": 100, "source_record": "Claude Opus 4.6 (high)|OpenClaw 2026.4.1", "source_date": "2026-04-13" }, { "id": "9bc5b20f4045d0804098", "model": "GPT-5.5 (xhigh)", "organization": "OpenAI", "configuration": "OpenClaw 2026.4.1", "score": 0.564, "raw_score": 56.4, "raw_scale": 100, "source_record": "GPT-5.5 (xhigh)|OpenClaw 2026.4.1", "source_date": "2026-05-15" }, { "id": "ab22a17021edf573c44b", "model": "GPT-5.4 (high)", "organization": "OpenAI", "configuration": "OpenClaw 2026.4.1", "score": 0.556, "raw_score": 55.6, "raw_scale": 100, "source_record": "GPT-5.4 (high)|OpenClaw 2026.4.1", "source_date": "2026-04-13" }, { "id": "ad7d5cba60dc212fc0e0", "model": "Gemini 3.1 Pro (high)", "organization": "Google", "configuration": "OpenClaw 2026.4.1", "score": 0.52, "raw_score": 52.0, "raw_scale": 100, "source_record": "Gemini 3.1 Pro (high)|OpenClaw 2026.4.1", "source_date": "2026-04-14" }, { "id": "36e0825b994647a348d4", "model": "Claude Sonnet 4.6 (high)", "organization": "Anthropic", "configuration": "OpenClaw 2026.4.1", "score": 0.519, "raw_score": 51.9, "raw_scale": 100, "source_record": "Claude Sonnet 4.6 (high)|OpenClaw 2026.4.1", "source_date": "2026-04-13" }, { "id": "9f77ea72dcb5dbef59d0", "model": "GLM 5.1 (enabled)", "organization": "Z AI", "configuration": "OpenClaw 2026.4.1; via OpenRouter, no round-trip reasoning context", "score": 0.505, "raw_score": 50.5, "raw_scale": 100, "source_record": "GLM 5.1 (enabled)|OpenClaw 2026.4.1; via OpenRouter, no round-trip reasoning context", "source_date": "2026-04-13" }, { "id": "1dc7e584fb95d60deb0e", "model": "Kimi-K2.5 (enabled)", "organization": "Moonshot AI", "configuration": "OpenClaw 2026.4.1; via OpenRouter, no round-trip reasoning context", "score": 0.34, "raw_score": 34.0, "raw_scale": 100, "source_record": "Kimi-K2.5 (enabled)|OpenClaw 2026.4.1; via OpenRouter, no round-trip reasoning context", "source_date": "2026-04-14" } ] }, { "id": "arc-agi-3", "name": "ARC-AGI-3", "group": "Agent 与工具", "unit": "score", "metric": "Human-normalized efficiency score", "protocol": { "en": "Semi-private interactive evaluation. Human-normalized action efficiency on a 0–100 scale, not task accuracy; excluded from accuracy saturation groups.", "zh": "半私有交互评测，保留 0–100 的人类归一化动作效率分数；不等同于任务准确率，不参加准确率饱和分组。" }, "source": { "data_url": "https://arcprize.org/media/data/leaderboard/v3.json", "sha256": "55c8e03e6962a2a873bc32284fb09ae5ac3306e39f428ca0ea07d453a4ca5cda", "retrieved_at": "2026-11-01T16:51:47.178925+00:00", "name": "ARC Prize", "url": "https://arcprize.org/leaderboard", "selector": "v3_Semi_Private; display=true; published LLM/reasoning configurations; score * 100", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "2ac44f09e42826ca0e18", "model": "GPT-6 Astra - Provider Adapter (High)", "organization": "OpenAI", "configuration": "CoT", "score": 99.94571232125699, "raw_score": 0.99945712321257, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-high-provider-adapter", "cost": 18816.63138 }, { "id": "5a42681df951456ffee3", "model": "GPT-6 Astra - Provider Adapter (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 98.55074288592294, "raw_score": 0.9855074288592295, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-max-provider-adapter", "cost": 17331.95327 }, { "id": "00c4679997fbf8d15642", "model": "GPT-6 Astra - Provider Adapter (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 98.44155844155843, "raw_score": 0.9844155844155843, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-xhigh-provider-adapter", "cost": 18147.191119999996 }, { "id": "43774fae4ead0d16530f", "model": "GPT-6 Astra - Provider Adapter (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 98.43861309255745, "raw_score": 0.9843861309255745, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-medium-provider-adapter", "cost": 19284.776630000004 }, { "id": "930f14528ab78c99e5ab", "model": "GPT-6 Astra - Provider Adapter (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 98.02838165829648, "raw_score": 0.9802838165829648, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-low-provider-adapter", "cost": 21297.922310000005 }, { "id": "b9db9fc8684dc6661e9d", "model": "Gemini 4 Argon - Provider Adapter (Low)", "organization": "Google", "configuration": "CoT", "score": 97.4591718377776, "raw_score": 0.974591718377776, "raw_scale": 0.01, "source_record": "google-gemini-4-argon-low-provider-adapter", "cost": 4770.148614 }, { "id": "b4ee5a87947d2e99d900", "model": "Gemini 4 Argon - Provider Adapter (Medium)", "organization": "Google", "configuration": "CoT", "score": 97.3448904168153, "raw_score": 0.9734489041681531, "raw_scale": 0.01, "source_record": "google-gemini-4-argon-medium-provider-adapter", "cost": 4981.202062 }, { "id": "8a3f4e96c948faa840ef", "model": "Gemini 4 Argon - Provider Adapter (High)", "organization": "Google", "configuration": "CoT", "score": 97.30432688837197, "raw_score": 0.9730432688837198, "raw_scale": 0.01, "source_record": "google-gemini-4-argon-high-provider-adapter", "cost": 4625.84128 }, { "id": "34a3b217f620a8e4e2db", "model": "GPT-6 Astra - Provider Adapter (None)", "organization": "OpenAI", "configuration": "CoT", "score": 96.72021831785531, "raw_score": 0.9672021831785531, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-none-provider-adapter", "cost": 23456.663940000002 }, { "id": "432fec132444aa5f1d93", "model": "GPT-6.1 Sol - Provider Adapter (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 96.37493342875182, "raw_score": 0.9637493342875182, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-xhigh-provider-adapter", "cost": 4360.206418 }, { "id": "af97975de829beef0b35", "model": "GPT-6.1 Sol - Provider Adapter (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 96.17739071933946, "raw_score": 0.9617739071933946, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-max-provider-adapter", "cost": 3816.796008 }, { "id": "2f7599d3759aa9c6c777", "model": "GPT-6.1 Sol - Provider Adapter (High)", "organization": "OpenAI", "configuration": "CoT", "score": 94.99362560053225, "raw_score": 0.9499362560053226, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-high-provider-adapter", "cost": 5403.981038 }, { "id": "d63064f33fa2922ee1e6", "model": "GPT-6.1 Sol - Provider Adapter (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 91.02346102095301, "raw_score": 0.9102346102095301, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-medium-provider-adapter", "cost": 5834.913158 }, { "id": "03d1626bd769de333eda", "model": "GPT-6.1 Sol - Provider Adapter (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 82.7870673049169, "raw_score": 0.8278706730491691, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-low-provider-adapter", "cost": 6622.199092 }, { "id": "e11d8ebf2d1587441460", "model": "Claude Sonnet 5.5 - Provider Adapter (XHigh)", "organization": "Anthropic", "configuration": "CoT", "score": 72.44170308802806, "raw_score": 0.7244170308802806, "raw_scale": 0.01, "source_record": "anthropic-claude-sonnet-5-5-xhigh-provider-adapter", "cost": 8169.902276 }, { "id": "88f48429bd39d2492872", "model": "GPT-6 Astra (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 62.712802100606275, "raw_score": 0.6271280210060628, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-max", "cost": 26097.501720000007 }, { "id": "960d3aa0d1d58c8001df", "model": "GPT-6 Astra (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 59.342954091460555, "raw_score": 0.5934295409146055, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-xhigh", "cost": 37317.38768 }, { "id": "d29dec4f370bc4a6c95c", "model": "GPT-6 Astra (High)", "organization": "OpenAI", "configuration": "CoT", "score": 54.8190516361223, "raw_score": 0.5481905163612231, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-high", "cost": 40704.722830000006 }, { "id": "97f2d39744d27a43065d", "model": "GPT-6.1 Sol (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 52.72904006820879, "raw_score": 0.5272904006820879, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-max", "cost": 7632.480056 }, { "id": "92bc321cbf49e327bf2e", "model": "Gemini 4 Argon (Low)", "organization": "Google", "configuration": "CoT", "score": 49.79401633943577, "raw_score": 0.4979401633943577, "raw_scale": 0.01, "source_record": "google-gemini-4-argon-low", "cost": 5820.531006 }, { "id": "275929d7ea1625be8603", "model": "Gemini 4 Argon (Medium)", "organization": "Google", "configuration": "CoT", "score": 46.265668402958575, "raw_score": 0.46265668402958576, "raw_scale": 0.01, "source_record": "google-gemini-4-argon-medium", "cost": 6179.253994 }, { "id": "8b07e0fd8e99bdeb3f66", "model": "GPT-6.1 Sol (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 39.92932040733721, "raw_score": 0.3992932040733721, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-xhigh", "cost": 9104.425822 }, { "id": "8782166651b943a59500", "model": "GPT-6 Astra (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 38.58757449733784, "raw_score": 0.3858757449733784, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-medium", "cost": 48090.34598999999 }, { "id": "132b4c94553fd0d96ede", "model": "Claude Sonnet 5.5 - Provider Adapter (High)", "organization": "Anthropic", "configuration": "CoT", "score": 35.47664601535195, "raw_score": 0.35476646015351954, "raw_scale": 0.01, "source_record": "anthropic-claude-sonnet-5-5-high-provider-adapter", "cost": 10244.13243 }, { "id": "022d4147ea0203146774", "model": "GPT-6 Astra (None)", "organization": "OpenAI", "configuration": "CoT", "score": 35.177880317424, "raw_score": 0.35177880317424, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-none", "cost": 49791.11317000001 }, { "id": "c56ebbd79d7a0c5c79cf", "model": "Gemini 3.8 Flash - Provider Adapter (High)", "organization": "Google", "configuration": "CoT", "score": 35.00424335075243, "raw_score": 0.35004243350752434, "raw_scale": 0.01, "source_record": "google-gemini-3-8-flash-high-provider-adapter", "cost": 4521.7983525 }, { "id": "556b644f6d06587817da", "model": "Claude Opus 5 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 30.159999999999997, "raw_score": 0.3016, "raw_scale": 0.01, "source_record": "anthropic-claude-opus-5-high", "cost": 20657.37 }, { "id": "5b50cdc5208933289fd0", "model": "GPT-6.1 Sol (High)", "organization": "OpenAI", "configuration": "CoT", "score": 26.717284844257062, "raw_score": 0.2671728484425706, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-high", "cost": 9757.61782 }, { "id": "7a7cec914c398dd0468a", "model": "Claude Sonnet 5.5 (XHigh)", "organization": "Anthropic", "configuration": "CoT", "score": 24.31040800668143, "raw_score": 0.24310408006681433, "raw_scale": 0.01, "source_record": "anthropic-claude-sonnet-5-5-xhigh", "cost": 7394.466286 }, { "id": "d8a28cf55a82bcc0fd38", "model": "Gemini 3.8 Flash - Provider Adapter (Medium)", "organization": "Google", "configuration": "CoT", "score": 24.202718056957025, "raw_score": 0.24202718056957026, "raw_scale": 0.01, "source_record": "google-gemini-3-8-flash-medium-provider-adapter", "cost": 3826.20432375 }, { "id": "54af15f9974a6082200c", "model": "GPT-6 Sol - Provider Adapter (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 23.028056734101494, "raw_score": 0.23028056734101496, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-max-provider-adapter", "cost": 8721.949674 }, { "id": "8bb37c7f90fbb8c18f36", "model": "GPT-6 Astra (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 17.452190401133805, "raw_score": 0.17452190401133805, "raw_scale": 0.01, "source_record": "openai-gpt-6-astra-low", "cost": 38166.47030000001 }, { "id": "9d038cca7783e4592bcb", "model": "Gemini 3.8 Flash - Provider Adapter (Low)", "organization": "Google", "configuration": "CoT", "score": 15.071935405559646, "raw_score": 0.15071935405559647, "raw_scale": 0.01, "source_record": "google-gemini-3-8-flash-low-provider-adapter", "cost": 3158.1084112500002 }, { "id": "0fbea5791eb486f0cc69", "model": "GPT-6.1 Sol (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 10.568764673656194, "raw_score": 0.10568764673656195, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-medium", "cost": 7369.782166 }, { "id": "44f9b77a07f6982a8f55", "model": "Gemini 3.8 Flash (High)", "organization": "Google", "configuration": "CoT", "score": 10.36627922392312, "raw_score": 0.10366279223923121, "raw_scale": 0.01, "source_record": "google-gemini-3-8-flash-high", "cost": 4398.2756895 }, { "id": "3c8a78646841f3cfae68", "model": "Grok 4.7 - Provider Adapter (High)", "organization": "xAI", "configuration": "CoT", "score": 10.047981644646995, "raw_score": 0.10047981644646996, "raw_scale": 0.01, "source_record": "xai-grok-4-7-high-provider-adapter", "cost": 4773.782536 }, { "id": "7cd5780e8d94bef62400", "model": "GPT-6 Sol - Provider Adapter (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 9.453334058864572, "raw_score": 0.09453334058864572, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-xhigh-provider-adapter", "cost": 6906.642396 }, { "id": "6ce20490203a83119d2b", "model": "Grok 4.7 - Provider Adapter (Medium)", "organization": "xAI", "configuration": "CoT", "score": 8.373637088154913, "raw_score": 0.08373637088154913, "raw_scale": 0.01, "source_record": "xai-grok-4-7-medium-provider-adapter", "cost": 5059.930398 }, { "id": "be551e1a6f34d6d063c9", "model": "GPT-5.6 Sol (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 7.779999999999999, "raw_score": 0.0778, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-sol-max", "cost": 25064.11 }, { "id": "4c19086c556ff751db6b", "model": "Grok 4.7 - Provider Adapter (XHigh)", "organization": "xAI", "configuration": "CoT", "score": 7.016486895838931, "raw_score": 0.0701648689583893, "raw_scale": 0.01, "source_record": "xai-grok-4-7-xhigh-provider-adapter", "cost": 4575.921274 }, { "id": "a0a23fbcdc116480e346", "model": "GPT-5.6 Sol (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 6.99, "raw_score": 0.0699, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-sol-xhigh", "cost": 19216.38 }, { "id": "b8487af4bea01a4bd35f", "model": "Gemini 3.8 Flash (Low)", "organization": "Google", "configuration": "CoT", "score": 5.989560493697216, "raw_score": 0.059895604936972165, "raw_scale": 0.01, "source_record": "google-gemini-3-8-flash-low", "cost": 2669.05677675 }, { "id": "d6cd4aea9bce530cc49b", "model": "GPT-6 Sol - Provider Adapter (High)", "organization": "OpenAI", "configuration": "CoT", "score": 5.855663760579018, "raw_score": 0.058556637605790184, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-high-provider-adapter", "cost": 5908.428836 }, { "id": "d7035f344d19e71aa92c", "model": "GPT-6 Sol - Provider Adapter (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 4.930246053628116, "raw_score": 0.04930246053628116, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-medium-provider-adapter", "cost": 5889.174868 }, { "id": "ad80bba6c1c32c51b0e1", "model": "Claude Sonnet 5.5 - Provider Adapter (Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 4.887554909249332, "raw_score": 0.048875549092493316, "raw_scale": 0.01, "source_record": "anthropic-claude-sonnet-5-5-medium-provider-adapter", "cost": 5250.163516 }, { "id": "bd1c0c7c32c0b3d290f1", "model": "GPT-6 Sol (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 4.623568186131146, "raw_score": 0.04623568186131146, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-max", "cost": 5554.493172 }, { "id": "3ca161cae85865931ef9", "model": "Gemini 3.8 Flash (Medium)", "organization": "Google", "configuration": "CoT", "score": 3.9878093830617267, "raw_score": 0.039878093830617266, "raw_scale": 0.01, "source_record": "google-gemini-3-8-flash-medium", "cost": 2727.66378825 }, { "id": "19a9fdeabdc46906a982", "model": "GPT-6.1 Sol (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 3.9243846587314173, "raw_score": 0.039243846587314175, "raw_scale": 0.01, "source_record": "openai-gpt-6-1-sol-low", "cost": 7110.559762 }, { "id": "768a2fab2ecb36afa73d", "model": "DeepSeek V4.1 Flash - Provider Adapter (Low)", "organization": "DeepSeek", "configuration": "CoT", "score": 3.264386796625269, "raw_score": 0.03264386796625269, "raw_scale": 0.01, "source_record": "deepseek-v4-1-flash-low-provider-adapter", "cost": 878.0226015 }, { "id": "95149cc37d682fecadab", "model": "DeepSeek V4.1 Flash - Provider Adapter (High)", "organization": "DeepSeek", "configuration": "CoT", "score": 3.1518486978853844, "raw_score": 0.031518486978853844, "raw_scale": 0.01, "source_record": "deepseek-v4-1-flash-high-provider-adapter", "cost": 1021.5134985 }, { "id": "809049451ea029eccc23", "model": "Claude Haiku 5.5 - Provider Adapter (Max)", "organization": "Anthropic", "configuration": "CoT", "score": 2.819365772805844, "raw_score": 0.02819365772805844, "raw_scale": 0.01, "source_record": "anthropic-claude-haiku-5-5-max-provider-adapter", "cost": 437.1406743 }, { "id": "5101e183964dc073b751", "model": "DeepSeek V4.1 Flash - Provider Adapter (Max)", "organization": "DeepSeek", "configuration": "CoT", "score": 2.6843916288028566, "raw_score": 0.026843916288028565, "raw_scale": 0.01, "source_record": "deepseek-v4-1-flash-max-provider-adapter", "cost": 1309.4644137 }, { "id": "411d002334d2986b5391", "model": "GPT-5.6 Sol (High)", "organization": "OpenAI", "configuration": "CoT", "score": 2.15, "raw_score": 0.0215, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-sol-high", "cost": 15176.07 }, { "id": "2e82481c7521128e4483", "model": "Grok 4.6 (XHigh)", "organization": "xAI", "configuration": "CoT", "score": 2.11, "raw_score": 0.0211, "raw_scale": 0.01, "source_record": "xai-grok-4-6-xhigh", "cost": 5612 }, { "id": "413faec415be6f532d49", "model": "Grok 4.7 (XHigh)", "organization": "xAI", "configuration": "CoT", "score": 1.8162526536637038, "raw_score": 0.01816252653663704, "raw_scale": 0.01, "source_record": "xai-grok-4-7-xhigh", "cost": 2731.577506 }, { "id": "009e7fba6bdf4864bbe0", "model": "GPT-6 Sol (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 1.7976269602783517, "raw_score": 0.017976269602783517, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-xhigh", "cost": 5105.623836 }, { "id": "96f4fa48d613d3f8ad9b", "model": "GPT-6 Sol - Provider Adapter (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 1.7619154311646181, "raw_score": 0.017619154311646183, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-low-provider-adapter", "cost": 4745.286212 }, { "id": "1788f7609e36bad08f93", "model": "Grok 4.7 (High)", "organization": "xAI", "configuration": "CoT", "score": 1.7332202934667684, "raw_score": 0.017332202934667685, "raw_scale": 0.01, "source_record": "xai-grok-4-7-high", "cost": 3741.649478 }, { "id": "54e3085d30ea73dc262d", "model": "Grok 4.7 (Medium)", "organization": "xAI", "configuration": "CoT", "score": 1.7056643767835524, "raw_score": 0.017056643767835524, "raw_scale": 0.01, "source_record": "xai-grok-4-7-medium", "cost": 3795.4947700000002 }, { "id": "c1c62af6d7218cb6d440", "model": "Claude Opus 4.8 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 1.52, "raw_score": 0.0152, "raw_scale": 0.01, "source_record": "anthropic-opus-4-8-high", "cost": 10000 }, { "id": "8ca8432077b3be30981a", "model": "GPT-6 Sol - Provider Adapter (None)", "organization": "OpenAI", "configuration": "CoT", "score": 1.2059164341155284, "raw_score": 0.012059164341155284, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-none-provider-adapter", "cost": 4233.770312 }, { "id": "b37a5bcf247974b48296", "model": "Claude Sonnet 5.5 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 1.203076515291301, "raw_score": 0.01203076515291301, "raw_scale": 0.01, "source_record": "anthropic-claude-sonnet-5-5-high", "cost": 5487.227968 }, { "id": "567654a9097d37b75f66", "model": "GPT-5.6 Sol (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 1.0699999999999998, "raw_score": 0.0107, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-sol-medium", "cost": 12971.17 }, { "id": "e25abe981c2f8527f198", "model": "GPT-6 Sol (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8994389911152361, "raw_score": 0.00899438991115236, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-high", "cost": 5072.447082 }, { "id": "9be3411ecd59e8477583", "model": "DeepSeek V4.1 Flash (Max)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.8516646657220222, "raw_score": 0.008516646657220222, "raw_scale": 0.01, "source_record": "deepseek-v4-1-flash-max", "cost": 1164.9106122 }, { "id": "11fdc7e72e882264b672", "model": "GPT-5.6 Terra (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.8, "raw_score": 0.008, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-terra-max", "cost": 7918.31 }, { "id": "77fa0b0225a7c0d8d315", "model": "Grok 4.7 - Provider Adapter (Low)", "organization": "xAI", "configuration": "CoT", "score": 0.7739582026314942, "raw_score": 0.007739582026314942, "raw_scale": 0.01, "source_record": "xai-grok-4-7-low-provider-adapter", "cost": 5753.573154 }, { "id": "b63f029908208cf65cc4", "model": "GPT-5.6 Terra (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.6499999999999999, "raw_score": 0.0065, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-terra-xhigh", "cost": 6804.04 }, { "id": "215cd2900daf27abd8a3", "model": "Claude Sonnet 5.5 - Provider Adapter (Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.6244567651378714, "raw_score": 0.006244567651378714, "raw_scale": 0.01, "source_record": "anthropic-claude-sonnet-5-5-low-provider-adapter", "cost": 4796.883118 }, { "id": "13c52eb3607ebf664e11", "model": "GPT-6 Luna - Provider Adapter (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5868386891554969, "raw_score": 0.0058683868915549685, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-max-provider-adapter", "cost": 237.1428722 }, { "id": "3920761b0bdd2ae07a4f", "model": "Anthropic Opus 4.6 (Max)", "organization": "Anthropic", "configuration": "CoT", "score": 0.51, "raw_score": 0.0051, "raw_scale": 0.01, "source_record": "anthropic-opus-4-6-max-effort", "cost": 8866.2 }, { "id": "a9cc7c4521bb37b2f7f5", "model": "GPT-6 Luna - Provider Adapter (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.5072886313635385, "raw_score": 0.005072886313635385, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-xhigh-provider-adapter", "cost": 223.17200050000002 }, { "id": "94ec53d3941e9136a4b2", "model": "GPT-5.6 Terra (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.49, "raw_score": 0.0049, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-terra-high", "cost": 5881.39 }, { "id": "8dd7bf216728fcb59495", "model": "DeepSeek V4.1 Flash (High)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.4846664966204641, "raw_score": 0.004846664966204641, "raw_scale": 0.01, "source_record": "deepseek-v4-1-flash-high", "cost": 1005.7964823 }, { "id": "018b9b6fce5885881450", "model": "DeepSeek V4.1 Flash - Provider Adapter (None)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.4830935313474596, "raw_score": 0.004830935313474596, "raw_scale": 0.01, "source_record": "deepseek-v4-1-flash-none-provider-adapter", "cost": 648.4115616 }, { "id": "6ccc112e494164371375", "model": "Claude Haiku 5.5 (Max)", "organization": "Anthropic", "configuration": "CoT", "score": 0.4513711877557703, "raw_score": 0.004513711877557703, "raw_scale": 0.01, "source_record": "anthropic-claude-haiku-5-5-max", "cost": 388.1060053 }, { "id": "e795bb77065b8e906b95", "model": "GPT-5.5 (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.43, "raw_score": 0.0043, "raw_scale": 0.01, "source_record": "openai-gpt-5-5-2026-04-23-high", "cost": 10000 }, { "id": "4e6bd67989b74db612f6", "model": "Gemini 3.1 Pro (Preview)", "organization": "Google", "configuration": "CoT", "score": 0.42, "raw_score": 0.0042, "raw_scale": 0.01, "source_record": "google-gemini-3-1-pro-preview", "cost": 2213.53 }, { "id": "7f7f1f8e5c63703b9a99", "model": "GPT-6 Sol (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.418612306617149, "raw_score": 0.00418612306617149, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-medium", "cost": 4863.795116 }, { "id": "3d78aa2b5c385711b789", "model": "Muse Spark Open - Provider Adapter (XHigh)", "organization": "Meta AI", "configuration": "CoT", "score": 0.41798234834263187, "raw_score": 0.004179823483426319, "raw_scale": 0.01, "source_record": "meta-muse-spark-open-xhigh-provider-adapter", "cost": 2995.96480425 }, { "id": "3b06ac765945063c23de", "model": "Claude Sonnet 5.5 (Low)", "organization": "Anthropic", "configuration": "CoT", "score": 0.4047513036164488, "raw_score": 0.004047513036164488, "raw_scale": 0.01, "source_record": "anthropic-claude-sonnet-5-5-low", "cost": 4756.680228 }, { "id": "b8fe27653f4ab0bbf881", "model": "Claude Sonnet 5.5 (Medium)", "organization": "Anthropic", "configuration": "CoT", "score": 0.3961083341784243, "raw_score": 0.003961083341784243, "raw_scale": 0.01, "source_record": "anthropic-claude-sonnet-5-5-medium", "cost": 4839.68249 }, { "id": "d45c605a3862b2031715", "model": "GPT-6 Luna - Provider Adapter (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.37847731587690087, "raw_score": 0.0037847731587690087, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-high-provider-adapter", "cost": 223.89528660000002 }, { "id": "ead5de54124aeafd1da3", "model": "DeepSeek V4.1 Flash (Low)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.34514247315937174, "raw_score": 0.0034514247315937177, "raw_scale": 0.01, "source_record": "deepseek-v4-1-flash-low", "cost": 820.6333452 }, { "id": "a332123c4d41ea5a8750", "model": "DeepSeek V4.1 Flash (None)", "organization": "DeepSeek", "configuration": "CoT", "score": 0.3431694493873915, "raw_score": 0.003431694493873915, "raw_scale": 0.01, "source_record": "deepseek-v4-1-flash-none", "cost": 734.7252066 }, { "id": "fa4719b3453045f8dca6", "model": "GPT-5.6 Sol (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.33, "raw_score": 0.0033, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-sol-low", "cost": 12805.6 }, { "id": "8c1563d40d75c4056d3c", "model": "Grok 4.5 (Medium)", "organization": "xAI", "configuration": "CoT", "score": 0.32, "raw_score": 0.0032, "raw_scale": 0.01, "source_record": "xai-grok-4-5-medium", "cost": 8458.3 }, { "id": "e72f18c0862b44a8eba4", "model": "GPT-6 Luna - Provider Adapter (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.31612009238177263, "raw_score": 0.003161200923817726, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-medium-provider-adapter", "cost": 213.71995940000002 }, { "id": "16f0d560909bc42dba3d", "model": "Grok 4.5 (High)", "organization": "xAI", "configuration": "CoT", "score": 0.3, "raw_score": 0.003, "raw_scale": 0.01, "source_record": "xai-grok-4-5-high", "cost": 6892.87 }, { "id": "9b538b67b51e63fd8e8d", "model": "GPT-6 Sol (None)", "organization": "OpenAI", "configuration": "CoT", "score": 0.2837742350643006, "raw_score": 0.0028377423506430063, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-none", "cost": 4655.549962 }, { "id": "c061e2168a85bbdd3822", "model": "Grok 4.5 (Low)", "organization": "xAI", "configuration": "CoT", "score": 0.26, "raw_score": 0.0026, "raw_scale": 0.01, "source_record": "xai-grok-4-5-low", "cost": 8718.48 }, { "id": "2fad3855b5e6e28f604a", "model": "Grok 4.7 (Low)", "organization": "xAI", "configuration": "CoT", "score": 0.24696281908456535, "raw_score": 0.0024696281908456537, "raw_scale": 0.01, "source_record": "xai-grok-4-7-low", "cost": 4814.88162 }, { "id": "afa3f435be55e10d9e36", "model": "GPT-6 Luna - Provider Adapter (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.2374807559887363, "raw_score": 0.002374807559887363, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-low-provider-adapter", "cost": 214.29109010000002 }, { "id": "37e44a3ea39a3e200ee3", "model": "GPT-5.4 (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.21, "raw_score": 0.0021, "raw_scale": 0.01, "source_record": "openai-gpt-5-4-2026-03-05-high", "cost": 5187.41 }, { "id": "e40f85b75d4b49d59614", "model": "GPT-6 Luna (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.19415284639354707, "raw_score": 0.0019415284639354709, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-medium", "cost": 241.2797313 }, { "id": "1043e5abc10417d0a6c6", "model": "GPT-6 Luna (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.18024216987988248, "raw_score": 0.0018024216987988248, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-high", "cost": 255.97928910000002 }, { "id": "590a59ff94fd4a4cac2f", "model": "GPT-5.6 Luna (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.18, "raw_score": 0.0018, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-luna-max", "cost": 3189.42 }, { "id": "a5726ee888dfd097e9af", "model": "Opus 4.7 (High)", "organization": "Anthropic", "configuration": "CoT", "score": 0.18, "raw_score": 0.0018, "raw_scale": 0.01, "source_record": "anthropic-opus-4-7-high", "cost": 10000 }, { "id": "7ed33f9a02fa1868094f", "model": "GPT-5.6 Luna (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.16999999999999998, "raw_score": 0.0017, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-luna-low", "cost": 2307.49 }, { "id": "203f3e30ee8c4f8f6b02", "model": "GPT-5.6 Luna (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.16999999999999998, "raw_score": 0.0017, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-luna-medium", "cost": 2379.13 }, { "id": "6d278812004c63d9e241", "model": "GPT-6 Luna (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.16413333960356602, "raw_score": 0.0016413333960356602, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-xhigh", "cost": 262.6844075 }, { "id": "9fde085da79f10d9b26f", "model": "Muse Spark Open (XHigh)", "organization": "Meta AI", "configuration": "CoT", "score": 0.13621973920554634, "raw_score": 0.0013621973920554635, "raw_scale": 0.01, "source_record": "meta-muse-spark-open-xhigh", "cost": 3908.42107525 }, { "id": "3524b0aa16e87d7e6de0", "model": "GPT-6 Sol (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.12649234250931635, "raw_score": 0.0012649234250931634, "raw_scale": 0.01, "source_record": "openai-gpt-6-sol-low", "cost": 4752.004208 }, { "id": "b541104d28447e2b9710", "model": "GPT-6 Luna (Max)", "organization": "OpenAI", "configuration": "CoT", "score": 0.10360997760319558, "raw_score": 0.0010360997760319559, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-max", "cost": 262.2377791 }, { "id": "21f6c06f670fb4ad76cd", "model": "GPT-5.6 Luna (High)", "organization": "OpenAI", "configuration": "CoT", "score": 0.1, "raw_score": 0.001, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-luna-high", "cost": 2482.44 }, { "id": "f7e8557e87a4ed1809f6", "model": "Grok 4.20 (Beta Reasoning)", "organization": "xAI", "configuration": "CoT", "score": 0.09, "raw_score": 0.0009, "raw_scale": 0.01, "source_record": "xai-grok-4-20-beta-0309-reasoning", "cost": 3775 }, { "id": "db537cb3526a5dcd3be8", "model": "GPT-5.6 Terra (Medium)", "organization": "OpenAI", "configuration": "CoT", "score": 0.08, "raw_score": 0.0008, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-terra-medium", "cost": 5895.31 }, { "id": "fd40ec4e7a3e099540b0", "model": "GPT-6 Luna - Provider Adapter (None)", "organization": "OpenAI", "configuration": "CoT", "score": 0.04644044435594966, "raw_score": 0.00046440444355949666, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-none-provider-adapter", "cost": 205.03171590000002 }, { "id": "25389e39b01c73d015ca", "model": "GPT-6 Luna (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.03291205821227175, "raw_score": 0.0003291205821227175, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-low", "cost": 228.8330075 }, { "id": "3b753720bf1e6dfa98d1", "model": "GPT-6 Luna (None)", "organization": "OpenAI", "configuration": "CoT", "score": 0.027919455042839488, "raw_score": 0.0002791945504283949, "raw_scale": 0.01, "source_record": "openai-gpt-6-luna-none", "cost": 229.77104200000002 }, { "id": "321ee56ecae91df6ca90", "model": "GPT-5.6 Luna (XHigh)", "organization": "OpenAI", "configuration": "CoT", "score": 0.02, "raw_score": 0.0002, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-luna-xhigh", "cost": 2801.59 }, { "id": "82431eb5d0ed95b9cb4c", "model": "GPT-5.6 Terra (Low)", "organization": "OpenAI", "configuration": "CoT", "score": 0.01, "raw_score": 0.0001, "raw_scale": 0.01, "source_record": "openai-gpt-5-6-terra-low", "cost": 5068.48 } ] }, { "id": "mmlu-deepseek-v4-base", "name": "MMLU (DeepSeek base, 5-shot)", "group": "知识与推理", "unit": "%", "metric": "Exact match", "protocol": { "en": "Base models; 5-shot exact match.", "zh": "Base 模型；5-shot 精确匹配。" }, "results": [ { "id": "f81667a1716461eaf38c", "model": "DeepSeek-V4-Pro-Base", "organization": "DeepSeek", "configuration": "Base; 5-shot", "score": 0.9009999999999999, "raw_score": 90.1, "raw_scale": 100, "source_record": "### Base Model|MMLU (EM)|DeepSeek-V4-Pro-Base", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "b60b5b8cc57168d4348d", "model": "DeepSeek-V4-Flash-Base", "organization": "DeepSeek", "configuration": "Base; 5-shot", "score": 0.887, "raw_score": 88.7, "raw_scale": 100, "source_record": "### Base Model|MMLU (EM)|DeepSeek-V4-Flash-Base", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "c3ef1bc8690f5811cb48", "model": "DeepSeek-V3.2-Base", "organization": "DeepSeek", "configuration": "Base; 5-shot", "score": 0.878, "raw_score": 87.8, "raw_scale": 100, "source_record": "### Base Model|MMLU (EM)|DeepSeek-V3.2-Base", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" } ], "source": { "name": "DeepSeek model report", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "data_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "sha256": "c4d714818a4d3333542edc7d38ea065825a0cf7aa8fea3605bbd1d1c18e4a610", "retrieved_at": "2026-11-01T17:19:58.924865+00:00", "selector": "### Base Model; MMLU (EM); 5-shot; Base model columns only", "kind": "model_report", "status": "current_snapshot" } }, { "id": "ceval-deepseek-v4-base", "name": "C-Eval (DeepSeek base, 5-shot)", "group": "知识与推理", "unit": "%", "metric": "Exact match", "protocol": { "en": "Base models; 5-shot exact match.", "zh": "Base 模型；5-shot 精确匹配。" }, "results": [ { "id": "b8b461ab2dddb6ceed50", "model": "DeepSeek-V4-Pro-Base", "organization": "DeepSeek", "configuration": "Base; 5-shot", "score": 0.9309999999999999, "raw_score": 93.1, "raw_scale": 100, "source_record": "### Base Model|C-Eval (EM)|DeepSeek-V4-Pro-Base", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "195c9d330c7e93b9a4c3", "model": "DeepSeek-V4-Flash-Base", "organization": "DeepSeek", "configuration": "Base; 5-shot", "score": 0.9209999999999999, "raw_score": 92.1, "raw_scale": 100, "source_record": "### Base Model|C-Eval (EM)|DeepSeek-V4-Flash-Base", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "608fda0c88137ec6491e", "model": "DeepSeek-V3.2-Base", "organization": "DeepSeek", "configuration": "Base; 5-shot", "score": 0.904, "raw_score": 90.4, "raw_scale": 100, "source_record": "### Base Model|C-Eval (EM)|DeepSeek-V3.2-Base", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" } ], "source": { "name": "DeepSeek model report", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "data_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "sha256": "c4d714818a4d3333542edc7d38ea065825a0cf7aa8fea3605bbd1d1c18e4a610", "retrieved_at": "2026-11-01T17:19:58.924865+00:00", "selector": "### Base Model; C-Eval (EM); 5-shot; Base model columns only", "kind": "model_report", "status": "current_snapshot" } }, { "id": "gsm8k-deepseek-v4-base", "name": "GSM8K (DeepSeek base, 8-shot)", "group": "数学与科学", "unit": "%", "metric": "Exact match", "protocol": { "en": "Base models; 8-shot exact match.", "zh": "Base 模型；8-shot 精确匹配。" }, "results": [ { "id": "e46d4647a077f79c30c0", "model": "DeepSeek-V4-Pro-Base", "organization": "DeepSeek", "configuration": "Base; 8-shot", "score": 0.9259999999999999, "raw_score": 92.6, "raw_scale": 100, "source_record": "### Base Model|GSM8K (EM)|DeepSeek-V4-Pro-Base", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "aa1e7d220c02e22ed74e", "model": "DeepSeek-V3.2-Base", "organization": "DeepSeek", "configuration": "Base; 8-shot", "score": 0.9109999999999999, "raw_score": 91.1, "raw_scale": 100, "source_record": "### Base Model|GSM8K (EM)|DeepSeek-V3.2-Base", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "c80057d969c793090d3a", "model": "DeepSeek-V4-Flash-Base", "organization": "DeepSeek", "configuration": "Base; 8-shot", "score": 0.9079999999999999, "raw_score": 90.8, "raw_scale": 100, "source_record": "### Base Model|GSM8K (EM)|DeepSeek-V4-Flash-Base", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" } ], "source": { "name": "DeepSeek model report", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "data_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "sha256": "c4d714818a4d3333542edc7d38ea065825a0cf7aa8fea3605bbd1d1c18e4a610", "retrieved_at": "2026-11-01T17:19:58.924865+00:00", "selector": "### Base Model; GSM8K (EM); 8-shot; Base model columns only", "kind": "model_report", "status": "current_snapshot" } }, { "id": "browsecomp-deepseek-v4", "name": "BrowseComp (DeepSeek report)", "group": "Agent 与工具", "unit": "%", "metric": "Pass@1", "protocol": { "en": "DeepSeek V4 model-report comparison, pass@1. Model and reasoning labels follow the report.", "zh": "DeepSeek V4 模型报告中的对比成绩，使用 pass@1；模型和推理模式名称沿用报告。" }, "results": [ { "id": "722df026cd16cc7c51c0", "model": "Gemini-3.1-Pro High", "organization": "Google", "configuration": "High", "score": 0.8590000000000001, "raw_score": 85.9, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|BrowseComp (Pass@1)|Gemini-3.1-Pro High", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "2fdd43c669f04c3bd609", "model": "Opus-4.6 Max", "organization": "Anthropic", "configuration": "Max", "score": 0.8370000000000001, "raw_score": 83.7, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|BrowseComp (Pass@1)|Opus-4.6 Max", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "36ab603d026878c39cb8", "model": "DS-V4-Pro Max", "organization": "DeepSeek", "configuration": "Max", "score": 0.8340000000000001, "raw_score": 83.4, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|BrowseComp (Pass@1)|DS-V4-Pro Max", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "6d10c13d038c51195d5f", "model": "K2.6 Thinking", "organization": "Moonshot AI", "configuration": "Thinking", "score": 0.8320000000000001, "raw_score": 83.2, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|BrowseComp (Pass@1)|K2.6 Thinking", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "6ce715ec128a2031918b", "model": "GPT-5.4 xHigh", "organization": "OpenAI", "configuration": "xHigh", "score": 0.8270000000000001, "raw_score": 82.7, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|BrowseComp (Pass@1)|GPT-5.4 xHigh", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "78166838eece6404f3a1", "model": "GLM-5.1 Thinking", "organization": "Z AI", "configuration": "Thinking", "score": 0.7929999999999999, "raw_score": 79.3, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|BrowseComp (Pass@1)|GLM-5.1 Thinking", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" } ], "source": { "name": "DeepSeek model report", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "data_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "sha256": "c4d714818a4d3333542edc7d38ea065825a0cf7aa8fea3605bbd1d1c18e4a610", "retrieved_at": "2026-11-01T17:19:58.924865+00:00", "selector": "#### DeepSeek-V4-Pro-Max vs Frontier Models; BrowseComp (Pass@1); excludes Comparison across Modes", "kind": "model_report", "status": "current_snapshot" } }, { "id": "chinese-simpleqa-deepseek-v4", "name": "Chinese SimpleQA (DeepSeek report)", "group": "知识与推理", "unit": "%", "metric": "Pass@1", "protocol": { "en": "DeepSeek V4 model-report comparison, pass@1. Model and reasoning labels follow the report.", "zh": "DeepSeek V4 模型报告中的对比成绩，使用 pass@1；模型和推理模式名称沿用报告。" }, "results": [ { "id": "3b4bf98f8616d2434936", "model": "Gemini-3.1-Pro High", "organization": "Google", "configuration": "High", "score": 0.8590000000000001, "raw_score": 85.9, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|Chinese-SimpleQA (Pass@1)|Gemini-3.1-Pro High", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "23dc853e1c7444e87188", "model": "DS-V4-Pro Max", "organization": "DeepSeek", "configuration": "Max", "score": 0.8440000000000001, "raw_score": 84.4, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|Chinese-SimpleQA (Pass@1)|DS-V4-Pro Max", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "b94a7ad0adaaa2cb740d", "model": "GPT-5.4 xHigh", "organization": "OpenAI", "configuration": "xHigh", "score": 0.768, "raw_score": 76.8, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|Chinese-SimpleQA (Pass@1)|GPT-5.4 xHigh", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "4fc0f2bb60991008b5b7", "model": "Opus-4.6 Max", "organization": "Anthropic", "configuration": "Max", "score": 0.764, "raw_score": 76.4, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|Chinese-SimpleQA (Pass@1)|Opus-4.6 Max", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "f9758b7b603f7ea09b30", "model": "K2.6 Thinking", "organization": "Moonshot AI", "configuration": "Thinking", "score": 0.759, "raw_score": 75.9, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|Chinese-SimpleQA (Pass@1)|K2.6 Thinking", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "403e62ee9b26a46e8835", "model": "GLM-5.1 Thinking", "organization": "Z AI", "configuration": "Thinking", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "#### DeepSeek-V4-Pro-Max vs Frontier Models|Chinese-SimpleQA (Pass@1)|GLM-5.1 Thinking", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" } ], "source": { "name": "DeepSeek model report", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "data_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/raw/main/README.md", "sha256": "c4d714818a4d3333542edc7d38ea065825a0cf7aa8fea3605bbd1d1c18e4a610", "retrieved_at": "2026-11-01T17:19:58.924865+00:00", "selector": "#### DeepSeek-V4-Pro-Max vs Frontier Models; Chinese-SimpleQA (Pass@1); excludes Comparison across Modes", "kind": "model_report", "status": "current_snapshot" } }, { "id": "paperbenchx", "name": "PaperBenchX (Pass rate)", "group": "数学与科学", "unit": "%", "metric": "Full", "source": { "data_url": "https://unipat.ai/benchmarks/PaperBenchX", "sha256": "9dd9d110f49c1c755998c180a542bd6aae6d22b16d805b46f2f28808c9edaf65", "retrieved_at": "2026-11-02T05:58:00.461107+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/PaperBenchX", "selector": "Overall results table: Full (%)", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "Pass rate across 93 tasks: a task passes only if its rubric score is strictly above 90% after clean replay. Missing outcomes count as zero.", "zh": "通过率：93 道任务中，经干净重放后评分严格高于 90% 的任务比例；缺失结果计零。" }, "results": [ { "id": "d45ba831c110150c9a45", "model": "GPT-6-Astra (Codex)", "organization": "OpenAI", "configuration": "Codex; 93 tasks", "score": 0.1398, "raw_score": 13.98, "raw_scale": 100, "source_record": "Full|GPT-6-Astra (Codex)", "average_turns": 348.7 }, { "id": "623631d1b5db829ee0f7", "model": "Fable 5.1", "organization": "Anthropic", "configuration": "93 tasks", "score": 0.129, "raw_score": 12.9, "raw_scale": 100, "source_record": "Full|Fable 5.1", "average_turns": 113.0 }, { "id": "226d56ea8e5b4df509b9", "model": "Fable 5", "organization": "Anthropic", "configuration": "93 tasks", "score": 0.1183, "raw_score": 11.83, "raw_scale": 100, "source_record": "Full|Fable 5", "average_turns": 112.0 }, { "id": "61a209a4e6971de8f94f", "model": "GPT-5.6-Sol (Codex)", "organization": "OpenAI", "configuration": "Codex; 93 tasks", "score": 0.0968, "raw_score": 9.68, "raw_scale": 100, "source_record": "Full|GPT-5.6-Sol (Codex)", "average_turns": 237.8 }, { "id": "c0f9761f17d068bcf566", "model": "DeepSeek-V4.1-Flash", "organization": "DeepSeek", "configuration": "93 tasks", "score": 0.0753, "raw_score": 7.53, "raw_scale": 100, "source_record": "Full|DeepSeek-V4.1-Flash", "average_turns": 372.6 }, { "id": "f81ee4871791eba3e864", "model": "GLM-5.3", "organization": "Z AI", "configuration": "93 tasks", "score": 0.0753, "raw_score": 7.53, "raw_scale": 100, "source_record": "Full|GLM-5.3", "average_turns": 290.8 }, { "id": "04422698e8e074adfe0c", "model": "Kimi-K3", "organization": "Moonshot AI", "configuration": "93 tasks", "score": 0.0753, "raw_score": 7.53, "raw_scale": 100, "source_record": "Full|Kimi-K3", "average_turns": 185.9 }, { "id": "09394ba0222fc41cbff0", "model": "Opus 5", "organization": "Anthropic", "configuration": "93 tasks", "score": 0.0753, "raw_score": 7.53, "raw_scale": 100, "source_record": "Full|Opus 5", "average_turns": 259.2 }, { "id": "fca53314bb262ea648bb", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "93 tasks", "score": 0.0538, "raw_score": 5.38, "raw_scale": 100, "source_record": "Full|Qwen3.8-Max", "average_turns": 234.4 }, { "id": "0d1f3cdad66335e72e57", "model": "HY3", "organization": "Tencent", "configuration": "93 tasks", "score": 0.0215, "raw_score": 2.15, "raw_scale": 100, "source_record": "Full|HY3", "average_turns": 197.4 } ] }, { "id": "paperbenchx-partial", "name": "PaperBenchX (Average completion)", "group": "数学与科学", "unit": "score", "metric": "Partial", "source": { "data_url": "https://unipat.ai/benchmarks/PaperBenchX", "sha256": "9dd9d110f49c1c755998c180a542bd6aae6d22b16d805b46f2f28808c9edaf65", "retrieved_at": "2026-11-02T05:58:00.461107+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/PaperBenchX", "selector": "Overall results table: Partial (%)", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "Average completion across all 93 tasks, measured by mean scientific rubric credit. Missing outcomes count as zero.", "zh": "平均完成度：全部 93 道任务的科学评分细则得分取平均；缺失结果计零。" }, "results": [ { "id": "a1fad1d594f1b7f79f4a", "model": "GPT-6-Astra (Codex)", "organization": "OpenAI", "configuration": "Codex; 93 tasks", "score": 59.08, "raw_score": 59.08, "raw_scale": 1, "source_record": "Partial|GPT-6-Astra (Codex)", "average_turns": 348.7 }, { "id": "be09a34e8d453caac289", "model": "Fable 5.1", "organization": "Anthropic", "configuration": "93 tasks", "score": 58.17, "raw_score": 58.17, "raw_scale": 1, "source_record": "Partial|Fable 5.1", "average_turns": 113.0 }, { "id": "dcb4cffcf8e51e8cc83d", "model": "Fable 5", "organization": "Anthropic", "configuration": "93 tasks", "score": 57.96, "raw_score": 57.96, "raw_scale": 1, "source_record": "Partial|Fable 5", "average_turns": 112.0 }, { "id": "fce0069448e3fd36ed8c", "model": "GPT-5.6-Sol (Codex)", "organization": "OpenAI", "configuration": "Codex; 93 tasks", "score": 55.07, "raw_score": 55.07, "raw_scale": 1, "source_record": "Partial|GPT-5.6-Sol (Codex)", "average_turns": 237.8 }, { "id": "5d73b30e677749fbda64", "model": "Opus 5", "organization": "Anthropic", "configuration": "93 tasks", "score": 52.52, "raw_score": 52.52, "raw_scale": 1, "source_record": "Partial|Opus 5", "average_turns": 259.2 }, { "id": "1a8c7e6253b6a80627f2", "model": "Kimi-K3", "organization": "Moonshot AI", "configuration": "93 tasks", "score": 49.24, "raw_score": 49.24, "raw_scale": 1, "source_record": "Partial|Kimi-K3", "average_turns": 185.9 }, { "id": "5bbdd2edac29f7467e26", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "93 tasks", "score": 46.76, "raw_score": 46.76, "raw_scale": 1, "source_record": "Partial|Qwen3.8-Max", "average_turns": 234.4 }, { "id": "d31adc55e2c01f682d90", "model": "DeepSeek-V4.1-Flash", "organization": "DeepSeek", "configuration": "93 tasks", "score": 45.47, "raw_score": 45.47, "raw_scale": 1, "source_record": "Partial|DeepSeek-V4.1-Flash", "average_turns": 372.6 }, { "id": "ddb8a8a62a069fcbffe3", "model": "GLM-5.3", "organization": "Z AI", "configuration": "93 tasks", "score": 40.78, "raw_score": 40.78, "raw_scale": 1, "source_record": "Partial|GLM-5.3", "average_turns": 290.8 }, { "id": "5e0e47433abcc81e8fe4", "model": "HY3", "organization": "Tencent", "configuration": "93 tasks", "score": 27.53, "raw_score": 27.53, "raw_scale": 1, "source_record": "Partial|HY3", "average_turns": 197.4 } ], "score_max": 100 }, { "id": "saas-bench-v1-1-resolved", "name": "SaaS-Bench v1.1 (Resolved)", "group": "Agent 与工具", "unit": "%", "metric": "Resolved", "source": { "data_url": "https://unipat.ai/benchmarks/SaaS-Bench", "sha256": "f8a0e865298de54790088605d8608e90e7179f00b9aecbc273e89b93b3b07fe6", "retrieved_at": "2026-11-02T05:58:00.461501+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/SaaS-Bench", "selector": "v1.1 results table: RS (%)", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "106 tasks across 23 SaaS systems. End-to-end task completion rate. Each model/harness pair is a separate result.", "zh": "23 个 SaaS 系统中的 106 道任务。使用端到端任务完成率。各模型与运行框架组合分别记录。" }, "results": [ { "id": "78ccf2943c4085880daf", "model": "Opus 5", "organization": "Anthropic", "configuration": "Claude Code; v1.1; 106 tasks", "score": 0.311, "raw_score": 31.1, "raw_scale": 100, "source_record": "RS|Opus 5|Claude Code" }, { "id": "4c2de7ddb5943e2e77ff", "model": "GPT-5.6 sol", "organization": "OpenAI", "configuration": "Codex; v1.1; 106 tasks", "score": 0.292, "raw_score": 29.2, "raw_scale": 100, "source_record": "RS|GPT-5.6 sol|Codex" }, { "id": "ac79e855347fc7afc43c", "model": "Pine Computer", "organization": "", "configuration": "Pine Computer Runtime (PCR); v1.1; 106 tasks", "score": 0.27399999999999997, "raw_score": 27.4, "raw_scale": 100, "source_record": "RS|Pine Computer|Pine Computer Runtime (PCR)" }, { "id": "e115408fb82cb7836df7", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Kimi-CLI; v1.1; 106 tasks", "score": 0.245, "raw_score": 24.5, "raw_scale": 100, "source_record": "RS|Kimi K3|Kimi-CLI" }, { "id": "4abbd1c3205c19ecbf5f", "model": "Qwen 3.8 max", "organization": "Alibaba", "configuration": "Qwen Code; v1.1; 106 tasks", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "RS|Qwen 3.8 max|Qwen Code" }, { "id": "82782252e792858bbbf0", "model": "Opus 5", "organization": "Anthropic", "configuration": "Browser-Use (Open-Source); v1.1; 106 tasks", "score": 0.217, "raw_score": 21.7, "raw_scale": 100, "source_record": "RS|Opus 5|Browser-Use (Open-Source)" }, { "id": "bab5aed08647a36febbf", "model": "GPT-5.6 sol", "organization": "OpenAI", "configuration": "Browser-Use (Open-Source); v1.1; 106 tasks", "score": 0.179, "raw_score": 17.9, "raw_scale": 100, "source_record": "RS|GPT-5.6 sol|Browser-Use (Open-Source)" }, { "id": "182fbcf46d94a2eb60f2", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Browser-Use (Open-Source); v1.1; 106 tasks", "score": 0.17, "raw_score": 17.0, "raw_scale": 100, "source_record": "RS|Kimi K3|Browser-Use (Open-Source)" }, { "id": "0f23fd81de413a4fd011", "model": "Qwen 3.8 max", "organization": "Alibaba", "configuration": "Browser-Use (Open-Source); v1.1; 106 tasks", "score": 0.085, "raw_score": 8.5, "raw_scale": 100, "source_record": "RS|Qwen 3.8 max|Browser-Use (Open-Source)" } ] }, { "id": "saas-bench-v1-1-checkpoint", "name": "SaaS-Bench v1.1 (Checkpoint)", "group": "Agent 与工具", "unit": "score", "metric": "Checkpoint", "source": { "data_url": "https://unipat.ai/benchmarks/SaaS-Bench", "sha256": "f8a0e865298de54790088605d8608e90e7179f00b9aecbc273e89b93b3b07fe6", "retrieved_at": "2026-11-02T05:58:00.461501+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/SaaS-Bench", "selector": "v1.1 results table: CS (%)", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "106 tasks across 23 SaaS systems. Partial checkpoint credit. Each model/harness pair is a separate result.", "zh": "23 个 SaaS 系统中的 106 道任务。使用检查点部分得分。各模型与运行框架组合分别记录。" }, "results": [ { "id": "1802688593a181d72b03", "model": "Pine Computer", "organization": "", "configuration": "Pine Computer Runtime (PCR); v1.1; 106 tasks", "score": 78.3, "raw_score": 78.3, "raw_scale": 1, "source_record": "CS|Pine Computer|Pine Computer Runtime (PCR)" }, { "id": "12a66cecdf636c8de3d8", "model": "Opus 5", "organization": "Anthropic", "configuration": "Claude Code; v1.1; 106 tasks", "score": 74.3, "raw_score": 74.3, "raw_scale": 1, "source_record": "CS|Opus 5|Claude Code" }, { "id": "1f468313cf66113cceac", "model": "GPT-5.6 sol", "organization": "OpenAI", "configuration": "Codex; v1.1; 106 tasks", "score": 71.1, "raw_score": 71.1, "raw_scale": 1, "source_record": "CS|GPT-5.6 sol|Codex" }, { "id": "8716ba28cd9f9a826a65", "model": "GPT-5.6 sol", "organization": "OpenAI", "configuration": "Browser-Use (Open-Source); v1.1; 106 tasks", "score": 69.8, "raw_score": 69.8, "raw_scale": 1, "source_record": "CS|GPT-5.6 sol|Browser-Use (Open-Source)" }, { "id": "134091bb74660a59c799", "model": "Qwen 3.8 max", "organization": "Alibaba", "configuration": "Qwen Code; v1.1; 106 tasks", "score": 65.9, "raw_score": 65.9, "raw_scale": 1, "source_record": "CS|Qwen 3.8 max|Qwen Code" }, { "id": "4a6308378470235a713c", "model": "Opus 5", "organization": "Anthropic", "configuration": "Browser-Use (Open-Source); v1.1; 106 tasks", "score": 64.7, "raw_score": 64.7, "raw_scale": 1, "source_record": "CS|Opus 5|Browser-Use (Open-Source)" }, { "id": "f668a9979acf67d8c142", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Kimi-CLI; v1.1; 106 tasks", "score": 56.4, "raw_score": 56.4, "raw_scale": 1, "source_record": "CS|Kimi K3|Kimi-CLI" }, { "id": "539f67a53be246a2610c", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Browser-Use (Open-Source); v1.1; 106 tasks", "score": 56.2, "raw_score": 56.2, "raw_scale": 1, "source_record": "CS|Kimi K3|Browser-Use (Open-Source)" }, { "id": "32902294c05b925dcb97", "model": "Qwen 3.8 max", "organization": "Alibaba", "configuration": "Browser-Use (Open-Source); v1.1; 106 tasks", "score": 31.1, "raw_score": 31.1, "raw_scale": 1, "source_record": "CS|Qwen 3.8 max|Browser-Use (Open-Source)" } ], "score_max": 100 }, { "id": "monthly-swebench-2026-06", "name": "Monthly-SWEBench (2026-06)", "group": "编程", "unit": "%", "metric": "Pass@1", "source": { "data_url": "https://unipat.ai/benchmarks/MonthlySWEBench", "sha256": "62ff6c25c82a4ab4f1e3b77aedc559ceafb373dda864a6e4a675057c183a2894", "retrieved_at": "2026-11-02T05:58:00.809814+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/MonthlySWEBench", "selector": "SWE_DATA.data.Overall.snapshots[2026-06]: passAt1; latest published batch", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "100 tasks, one attempt; 2026-06 release. The publisher counts missing/unclean required runs as zero under its provisional publication policy.", "zh": "2026-06 批次，100 道任务、单次尝试。官方临时发布口径将缺失或尚未完成干净重跑的必需记录计零。" }, "results": [ { "id": "cd609d8ea71ab6bd471a", "model": "GPT-5.5-High", "organization": "OpenAI", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.76, "raw_score": 76.0, "raw_scale": 100, "source_record": "2026-06|Overall|10", "task_count": 100, "passed_tasks": 76, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "5755c27aa4b9f3631bb5", "model": "Claude-Opus-4.8-XHigh", "organization": "Anthropic", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.69, "raw_score": 69.0, "raw_scale": 100, "source_record": "2026-06|Overall|12", "task_count": 100, "passed_tasks": 69, "unfinished_count": 3, "unfinished_reasons": { "timeout:AgentTimeoutError": 1, "zero_filled:missing_result_json": 2 } }, { "id": "7ad46f32b086f057e02b", "model": "GLM-5.2-Max", "organization": "Zhipu AI", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.65, "raw_score": 65.0, "raw_scale": 100, "source_record": "2026-06|Overall|11", "task_count": 100, "passed_tasks": 65, "unfinished_count": 2, "unfinished_reasons": { "timeout:AgentTimeoutError": 2 } }, { "id": "426eb4e666f21ea6a956", "model": "MiniMax-M3-Adaptive", "organization": "MiniMax", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.58, "raw_score": 58.0, "raw_scale": 100, "source_record": "2026-06|Overall|1", "task_count": 100, "passed_tasks": 58, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "4bfd9ee81710889c7d6c", "model": "DeepSeek-V4-Pro-High", "organization": "DeepSeek", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.57, "raw_score": 57.0, "raw_scale": 100, "source_record": "2026-06|Overall|4", "task_count": 100, "passed_tasks": 57, "unfinished_count": 1, "unfinished_reasons": { "timeout:AgentTimeoutError": 1 } }, { "id": "f367eeab86ac0e6680a4", "model": "Doubao-Seed-2.1-Pro-High", "organization": "ByteDance", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.53, "raw_score": 53.0, "raw_scale": 100, "source_record": "2026-06|Overall|6", "task_count": 100, "passed_tasks": 53, "unfinished_count": 15, "unfinished_reasons": { "timeout:AgentTimeoutError": 5, "zero_filled:missing_result_json": 10 } }, { "id": "f912f390e05dba1bafc7", "model": "Qwen-3.7-Max-Thinking", "organization": "Alibaba", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.51, "raw_score": 51.0, "raw_scale": 100, "source_record": "2026-06|Overall|8", "task_count": 100, "passed_tasks": 51, "unfinished_count": 2, "unfinished_reasons": { "zero_filled:missing_result_json": 2 } }, { "id": "a4682b4242b5af61f4cd", "model": "Qwen-3.7-Plus-Thinking", "organization": "Alibaba", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.47, "raw_score": 47.0, "raw_scale": 100, "source_record": "2026-06|Overall|9", "task_count": 100, "passed_tasks": 47, "unfinished_count": 2, "unfinished_reasons": { "timeout:AgentTimeoutError": 1, "zero_filled:missing_result_json": 1 } }, { "id": "aa75d25423d8af9d19b5", "model": "DeepSeek-V4-Flash-High", "organization": "DeepSeek", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.45, "raw_score": 45.0, "raw_scale": 100, "source_record": "2026-06|Overall|5", "task_count": 100, "passed_tasks": 45, "unfinished_count": 2, "unfinished_reasons": { "zero_filled:missing_result_json": 2 } }, { "id": "5d2c1fff99dca3fe1a1e", "model": "Hunyuan-HY3-Preview-High", "organization": "Tencent", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.38, "raw_score": 38.0, "raw_scale": 100, "source_record": "2026-06|Overall|2", "task_count": 100, "passed_tasks": 38, "unfinished_count": 1, "unfinished_reasons": { "timeout:AgentTimeoutError": 1 } }, { "id": "b4bf23ee6dbd3ed7db3b", "model": "Gemini-3.5-Flash-High", "organization": "Google", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.29, "raw_score": 29.0, "raw_scale": 100, "source_record": "2026-06|Overall|7", "task_count": 100, "passed_tasks": 29, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "1890bb3297d127280ead", "model": "Step-3.7-Flash-High", "organization": "StepFun", "configuration": "OpenHands; 2026-06; 100 tasks", "score": 0.22, "raw_score": 22.0, "raw_scale": 100, "source_record": "2026-06|Overall|3", "task_count": 100, "passed_tasks": 22, "unfinished_count": 0, "unfinished_reasons": {} } ] }, { "id": "roadmapbench-resolved", "name": "RoadmapBench (Resolved)", "group": "编程", "unit": "%", "metric": "Resolved", "source": { "data_url": "https://unipat.ai/benchmarks/RoadmapBench", "sha256": "20e6f355dd6f40c4e10ab2cc8f89308d8680482875074cd83f31841c67747ec3", "retrieved_at": "2026-11-02T05:58:00.625043+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/RoadmapBench", "selector": "Leaderboard table: Resolved %", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://arxiv.org/html/2605.15846v2", "sha256": "f8d46825b66278370841e87477dd02312e9e63d3f8b816be48c17a4c92ddc1c7", "retrieved_at": "2026-11-02T05:59:46.673715+00:00" } ] }, "protocol": { "en": "OpenHands; 115 version-upgrade tasks. Task passes only when all targets pass.", "zh": "OpenHands；115 道版本升级任务。所有目标均通过时才算完成任务。" }, "results": [ { "id": "388557bbc99df16009f0", "model": "Claude-Opus-4.7", "organization": "Anthropic", "configuration": "OpenHands; 115 tasks", "score": 0.391, "raw_score": 39.1, "raw_scale": 100, "source_record": "Resolved %|Claude-Opus-4.7 Anthropic|OpenHands" }, { "id": "6ae3f95e57290634286a", "model": "Claude-Opus-4.6", "organization": "Anthropic", "configuration": "OpenHands; 115 tasks", "score": 0.322, "raw_score": 32.2, "raw_scale": 100, "source_record": "Resolved %|Claude-Opus-4.6 Anthropic|OpenHands" }, { "id": "b2a9643dd8d188e913b0", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "OpenHands; 115 tasks", "score": 0.29600000000000004, "raw_score": 29.6, "raw_scale": 100, "source_record": "Resolved %|GPT-5.4 OpenAI|OpenHands" }, { "id": "d618f3e7ed32c188032c", "model": "Gemini-3.1-Pro", "organization": "Google", "configuration": "OpenHands; 115 tasks", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "Resolved %|Gemini-3.1-Pro Google|OpenHands" }, { "id": "3fb4b9a0f30367e587dd", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "OpenHands; 115 tasks", "score": 0.183, "raw_score": 18.3, "raw_scale": 100, "source_record": "Resolved %|DeepSeek-V4-Pro DeepSeek|OpenHands" }, { "id": "d96c4de726b3e48e8810", "model": "GLM-5.1", "organization": "Zhipu AI", "configuration": "OpenHands; 115 tasks", "score": 0.183, "raw_score": 18.3, "raw_scale": 100, "source_record": "Resolved %|GLM-5.1 Zhipu AI|OpenHands" }, { "id": "25e638c021cd30cd809f", "model": "Kimi-K2.6", "organization": "Moonshot AI", "configuration": "OpenHands; 115 tasks", "score": 0.14800000000000002, "raw_score": 14.8, "raw_scale": 100, "source_record": "Resolved %|Kimi-K2.6 Moonshot AI|OpenHands" }, { "id": "c11c8297576b260d7451", "model": "Mimo-V2.5-Pro", "organization": "Xiaomi", "configuration": "OpenHands; 115 tasks", "score": 0.139, "raw_score": 13.9, "raw_scale": 100, "source_record": "Resolved %|Mimo-V2.5-Pro Xiaomi|OpenHands" }, { "id": "5d84031476f8ed37d86d", "model": "Qwen3.6-Plus", "organization": "Alibaba", "configuration": "OpenHands; 115 tasks", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "Resolved %|Qwen3.6-Plus Alibaba|OpenHands" }, { "id": "9ad6f01bc101e6596e24", "model": "Kimi-K2.5", "organization": "Moonshot AI", "configuration": "OpenHands; 115 tasks", "score": 0.113, "raw_score": 11.3, "raw_scale": 100, "source_record": "Resolved %|Kimi-K2.5 Moonshot AI|OpenHands" }, { "id": "905e2d46db04837344c2", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "OpenHands; 115 tasks", "score": 0.10400000000000001, "raw_score": 10.4, "raw_scale": 100, "source_record": "Resolved %|MiniMax-M2.7 MiniMax|OpenHands" }, { "id": "badee55b419a9dec357c", "model": "Qwen3.5-397B", "organization": "Alibaba", "configuration": "OpenHands; 115 tasks", "score": 0.096, "raw_score": 9.6, "raw_scale": 100, "source_record": "Resolved %|Qwen3.5-397B Alibaba|OpenHands" }, { "id": "dfa0de56b0838d936142", "model": "Seed-2.0-Pro", "organization": "ByteDance", "configuration": "OpenHands; 115 tasks", "score": 0.052000000000000005, "raw_score": 5.2, "raw_scale": 100, "source_record": "Table 2|Seed-2.0-Pro|OpenHands|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.16" } ] }, { "id": "roadmapbench-completion", "name": "RoadmapBench (Completion)", "group": "编程", "unit": "score", "metric": "Completion", "source": { "data_url": "https://unipat.ai/benchmarks/RoadmapBench", "sha256": "20e6f355dd6f40c4e10ab2cc8f89308d8680482875074cd83f31841c67747ec3", "retrieved_at": "2026-11-02T05:58:00.625043+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/RoadmapBench", "selector": "Leaderboard table: Completion Score", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://arxiv.org/html/2605.15846v2", "sha256": "f8d46825b66278370841e87477dd02312e9e63d3f8b816be48c17a4c92ddc1c7", "retrieved_at": "2026-11-02T05:59:46.673715+00:00" } ] }, "protocol": { "en": "OpenHands; 115 version-upgrade tasks. Original 0–1 phase-completion score.", "zh": "OpenHands；115 道版本升级任务。保留 0–1 的阶段完成分数。" }, "results": [ { "id": "ee623a896f600deb5d59", "model": "Claude-Opus-4.7", "organization": "Anthropic", "configuration": "OpenHands; 115 tasks", "score": 0.692, "raw_score": 0.692, "raw_scale": 1, "source_record": "Completion Score|Claude-Opus-4.7 Anthropic|OpenHands" }, { "id": "5000df9bdf3728a187a8", "model": "Claude-Opus-4.6", "organization": "Anthropic", "configuration": "OpenHands; 115 tasks", "score": 0.627, "raw_score": 0.627, "raw_scale": 1, "source_record": "Completion Score|Claude-Opus-4.6 Anthropic|OpenHands" }, { "id": "b8364aa03fa99750097f", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "OpenHands; 115 tasks", "score": 0.497, "raw_score": 0.497, "raw_scale": 1, "source_record": "Completion Score|GPT-5.4 OpenAI|OpenHands" }, { "id": "9c76eecc533bce2e4127", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "OpenHands; 115 tasks", "score": 0.486, "raw_score": 0.486, "raw_scale": 1, "source_record": "Completion Score|DeepSeek-V4-Pro DeepSeek|OpenHands" }, { "id": "4b2799380687d07a67c4", "model": "GLM-5.1", "organization": "Zhipu AI", "configuration": "OpenHands; 115 tasks", "score": 0.453, "raw_score": 0.453, "raw_scale": 1, "source_record": "Completion Score|GLM-5.1 Zhipu AI|OpenHands" }, { "id": "30b50871949820400fe4", "model": "Mimo-V2.5-Pro", "organization": "Xiaomi", "configuration": "OpenHands; 115 tasks", "score": 0.44, "raw_score": 0.44, "raw_scale": 1, "source_record": "Completion Score|Mimo-V2.5-Pro Xiaomi|OpenHands" }, { "id": "63f1c7fc00c53c0f3c39", "model": "Gemini-3.1-Pro", "organization": "Google", "configuration": "OpenHands; 115 tasks", "score": 0.439, "raw_score": 0.439, "raw_scale": 1, "source_record": "Completion Score|Gemini-3.1-Pro Google|OpenHands" }, { "id": "0463b840dcbaa155913e", "model": "Kimi-K2.6", "organization": "Moonshot AI", "configuration": "OpenHands; 115 tasks", "score": 0.432, "raw_score": 0.432, "raw_scale": 1, "source_record": "Completion Score|Kimi-K2.6 Moonshot AI|OpenHands" }, { "id": "fb878690318cac487f9d", "model": "Qwen3.6-Plus", "organization": "Alibaba", "configuration": "OpenHands; 115 tasks", "score": 0.424, "raw_score": 0.424, "raw_scale": 1, "source_record": "Completion Score|Qwen3.6-Plus Alibaba|OpenHands" }, { "id": "b45427d80ebbaa91455a", "model": "Qwen3.5-397B", "organization": "Alibaba", "configuration": "OpenHands; 115 tasks", "score": 0.383, "raw_score": 0.383, "raw_scale": 1, "source_record": "Completion Score|Qwen3.5-397B Alibaba|OpenHands" }, { "id": "1b9b291133d010e31467", "model": "Kimi-K2.5", "organization": "Moonshot AI", "configuration": "OpenHands; 115 tasks", "score": 0.378, "raw_score": 0.378, "raw_scale": 1, "source_record": "Completion Score|Kimi-K2.5 Moonshot AI|OpenHands" }, { "id": "7ba08275891f79c5b2c5", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "OpenHands; 115 tasks", "score": 0.332, "raw_score": 0.332, "raw_scale": 1, "source_record": "Completion Score|MiniMax-M2.7 MiniMax|OpenHands" }, { "id": "68da9c8374abda08c57a", "model": "Seed-2.0-Pro", "organization": "ByteDance", "configuration": "OpenHands; 115 tasks", "score": 0.177, "raw_score": 0.177, "raw_scale": 1, "source_record": "Table 2|Seed-2.0-Pro|OpenHands|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.16" } ], "score_max": 1 }, { "id": "evocodebench-dataset", "name": "EvoCode-Bench (Dataset score)", "group": "编程", "unit": "score", "metric": "Dataset score", "source": { "data_url": "https://raw.githubusercontent.com/UniPat-AI/EvoCodeBench/main/README.md", "sha256": "875d99d5960c0299135ae519e9523114f1d9110da9c49d58de9b52fd03ad01f9", "retrieved_at": "2026-11-02T05:58:07.555555+00:00", "name": "UniPat AI", "url": "https://github.com/UniPat-AI/EvoCodeBench", "selector": "Current 2026-06-25 clean release table: Dataset score; excludes legacy MT@4", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://raw.githubusercontent.com/UniPat-AI/EvoCodeBench/main/evaluation/sweeps/sweep_2026-06_single_shot.csv", "sha256": "e57e2dbc515be48356590b0b6abb22ddc97e0f7e46098b363fffac691d9c20a9", "retrieved_at": "2026-11-02T05:58:07.365706+00:00" } ] }, "protocol": { "en": "Clean 2026-06-25 release; terminus-2, one attempt, 26 tasks. Mean per-task fraction of passing rounds, scaled to 100.", "zh": "2026-06-25 干净重跑批次；terminus-2、单次尝试、26 道任务。先计算每道任务的轮次通过比例，再平均并乘以 100。" }, "results": [ { "id": "13c96e88ca3422526454", "model": "Claude-Opus-4.8", "organization": "Anthropic", "configuration": "terminus-2; effort xhigh; one attempt; clean 2026-06-25 release", "score": 59.1, "raw_score": 59.1, "raw_scale": 1, "source_record": "dataset|Claude-Opus-4.8", "passed_tasks": 9, "task_count": 26 }, { "id": "1e0cca57724218d40a13", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 29.5, "raw_score": 29.5, "raw_scale": 1, "source_record": "dataset|GPT-5.5", "passed_tasks": 0, "task_count": 26 }, { "id": "085db6677da170234dc6", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "terminus-2; thinking adaptive; one attempt; clean 2026-06-25 release", "score": 23.4, "raw_score": 23.4, "raw_scale": 1, "source_record": "dataset|MiniMax-M3", "passed_tasks": 2, "task_count": 26 }, { "id": "313bcf63094175a54133", "model": "GLM-5.2", "organization": "Z AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 16.2, "raw_score": 16.2, "raw_scale": 1, "source_record": "dataset|GLM-5.2", "passed_tasks": 1, "task_count": 26 }, { "id": "f7e0049ba4b8e58966a9", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 14.1, "raw_score": 14.1, "raw_scale": 1, "source_record": "dataset|DeepSeek-V4-Pro", "passed_tasks": 1, "task_count": 26 }, { "id": "4be6ea41623e3ea58a1d", "model": "Kimi-K2.6", "organization": "Moonshot AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 13.2, "raw_score": 13.2, "raw_scale": 1, "source_record": "dataset|Kimi-K2.6", "passed_tasks": 0, "task_count": 26 }, { "id": "77a323c81645ec2070dd", "model": "DeepSeek-V4-Flash", "organization": "DeepSeek", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 12.2, "raw_score": 12.2, "raw_scale": 1, "source_record": "dataset|DeepSeek-V4-Flash", "passed_tasks": 0, "task_count": 26 }, { "id": "b53dba54450a420af6bf", "model": "Qwen3.7-Max", "organization": "Alibaba", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 11.9, "raw_score": 11.9, "raw_scale": 1, "source_record": "dataset|Qwen3.7-Max", "passed_tasks": 0, "task_count": 26 }, { "id": "eb0659bf39677e3a4c96", "model": "Qwen3.6-Plus", "organization": "Alibaba", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 9.7, "raw_score": 9.7, "raw_scale": 1, "source_record": "dataset|Qwen3.6-Plus", "passed_tasks": 0, "task_count": 26 }, { "id": "29f9fd9cd819161ce91c", "model": "Kimi-K2.7-Code", "organization": "Moonshot AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 7.8, "raw_score": 7.8, "raw_scale": 1, "source_record": "dataset|Kimi-K2.7-Code", "passed_tasks": 0, "task_count": 26 }, { "id": "1121b9f7011519cddbb1", "model": "Doubao-Seed-2.1-Pro", "organization": "", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 6.7, "raw_score": 6.7, "raw_scale": 1, "source_record": "dataset|Doubao-Seed-2.1-Pro", "passed_tasks": 0, "task_count": 26 }, { "id": "cda96d6fc91817f74884", "model": "GLM-5.1", "organization": "Z AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 5.9, "raw_score": 5.9, "raw_scale": 1, "source_record": "dataset|GLM-5.1", "passed_tasks": 0, "task_count": 26 }, { "id": "52d2de1b4aad68ae4ec4", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "terminus-2; reasoning split; one attempt; clean 2026-06-25 release", "score": 5.1, "raw_score": 5.1, "raw_scale": 1, "source_record": "dataset|MiniMax-M2.7", "passed_tasks": 0, "task_count": 26 } ], "score_max": 100 }, { "id": "evocodebench-case", "name": "EvoCode-Bench (Case score)", "group": "编程", "unit": "score", "metric": "Case score", "source": { "data_url": "https://raw.githubusercontent.com/UniPat-AI/EvoCodeBench/main/README.md", "sha256": "875d99d5960c0299135ae519e9523114f1d9110da9c49d58de9b52fd03ad01f9", "retrieved_at": "2026-11-02T05:58:07.555555+00:00", "name": "UniPat AI", "url": "https://github.com/UniPat-AI/EvoCodeBench", "selector": "Current 2026-06-25 clean release table: Case score; excludes legacy MT@4", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://raw.githubusercontent.com/UniPat-AI/EvoCodeBench/main/evaluation/sweeps/sweep_2026-06_single_shot.csv", "sha256": "e57e2dbc515be48356590b0b6abb22ddc97e0f7e46098b363fffac691d9c20a9", "retrieved_at": "2026-11-02T05:58:07.365706+00:00" } ] }, "protocol": { "en": "Clean 2026-06-25 release; terminus-2, one attempt, 26 tasks. Mean test-case credit across rounds and tasks, scaled to 100.", "zh": "2026-06-25 干净重跑批次；terminus-2、单次尝试、26 道任务。各轮测试用例得分按任务平均，换算为 0–100 分。" }, "results": [ { "id": "d96663162463c4025ed2", "model": "Claude-Opus-4.8", "organization": "Anthropic", "configuration": "terminus-2; effort xhigh; one attempt; clean 2026-06-25 release", "score": 96.6, "raw_score": 96.6, "raw_scale": 1, "source_record": "case|Claude-Opus-4.8", "passed_tasks": 9, "task_count": 26 }, { "id": "4f7768a35f1935a992a1", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 81.8, "raw_score": 81.8, "raw_scale": 1, "source_record": "case|GPT-5.5", "passed_tasks": 0, "task_count": 26 }, { "id": "b77dcc674618a7804c3f", "model": "Qwen3.6-Plus", "organization": "Alibaba", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 67.7, "raw_score": 67.7, "raw_scale": 1, "source_record": "case|Qwen3.6-Plus", "passed_tasks": 0, "task_count": 26 }, { "id": "2f484716a74a9029264d", "model": "Qwen3.7-Max", "organization": "Alibaba", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 67.4, "raw_score": 67.4, "raw_scale": 1, "source_record": "case|Qwen3.7-Max", "passed_tasks": 0, "task_count": 26 }, { "id": "958dc156423507d4c191", "model": "Kimi-K2.6", "organization": "Moonshot AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 65.7, "raw_score": 65.7, "raw_scale": 1, "source_record": "case|Kimi-K2.6", "passed_tasks": 0, "task_count": 26 }, { "id": "6f4a57002c7fa97b1678", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 61.7, "raw_score": 61.7, "raw_scale": 1, "source_record": "case|DeepSeek-V4-Pro", "passed_tasks": 1, "task_count": 26 }, { "id": "39b1beca812d1dbd7d17", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "terminus-2; thinking adaptive; one attempt; clean 2026-06-25 release", "score": 61.5, "raw_score": 61.5, "raw_scale": 1, "source_record": "case|MiniMax-M3", "passed_tasks": 2, "task_count": 26 }, { "id": "c79632cf1d4cbaa41bbe", "model": "DeepSeek-V4-Flash", "organization": "DeepSeek", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 58.9, "raw_score": 58.9, "raw_scale": 1, "source_record": "case|DeepSeek-V4-Flash", "passed_tasks": 0, "task_count": 26 }, { "id": "d1bb02b70f208eb1765f", "model": "GLM-5.1", "organization": "Z AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 52.5, "raw_score": 52.5, "raw_scale": 1, "source_record": "case|GLM-5.1", "passed_tasks": 0, "task_count": 26 }, { "id": "1aa72bdb7f676151b134", "model": "GLM-5.2", "organization": "Z AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 47.5, "raw_score": 47.5, "raw_scale": 1, "source_record": "case|GLM-5.2", "passed_tasks": 1, "task_count": 26 }, { "id": "5c7017a235c66dcf2b47", "model": "Kimi-K2.7-Code", "organization": "Moonshot AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 45.4, "raw_score": 45.4, "raw_scale": 1, "source_record": "case|Kimi-K2.7-Code", "passed_tasks": 0, "task_count": 26 }, { "id": "561ae92208036a0cf870", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "terminus-2; reasoning split; one attempt; clean 2026-06-25 release", "score": 44.9, "raw_score": 44.9, "raw_scale": 1, "source_record": "case|MiniMax-M2.7", "passed_tasks": 0, "task_count": 26 }, { "id": "93883ceb796bdd463fe5", "model": "Doubao-Seed-2.1-Pro", "organization": "", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 42.0, "raw_score": 42.0, "raw_scale": 1, "source_record": "case|Doubao-Seed-2.1-Pro", "passed_tasks": 0, "task_count": 26 } ], "score_max": 100 }, { "id": "evocodebench-perfect", "name": "EvoCode-Bench (Perfect tasks)", "group": "编程", "unit": "%", "metric": "Perfect tasks", "source": { "data_url": "https://raw.githubusercontent.com/UniPat-AI/EvoCodeBench/main/README.md", "sha256": "875d99d5960c0299135ae519e9523114f1d9110da9c49d58de9b52fd03ad01f9", "retrieved_at": "2026-11-02T05:58:07.555555+00:00", "name": "UniPat AI", "url": "https://github.com/UniPat-AI/EvoCodeBench", "selector": "Current 2026-06-25 clean release table: Perfect tasks; excludes legacy MT@4", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://raw.githubusercontent.com/UniPat-AI/EvoCodeBench/main/evaluation/sweeps/sweep_2026-06_single_shot.csv", "sha256": "e57e2dbc515be48356590b0b6abb22ddc97e0f7e46098b363fffac691d9c20a9", "retrieved_at": "2026-11-02T05:58:07.365706+00:00" } ] }, "protocol": { "en": "Clean 2026-06-25 release; terminus-2, one attempt, 26 tasks. Fraction of tasks passing every round.", "zh": "2026-06-25 干净重跑批次；terminus-2、单次尝试、26 道任务。每一轮均通过的完整任务比例。" }, "results": [ { "id": "40c2d0e60902165c20bd", "model": "Claude-Opus-4.8", "organization": "Anthropic", "configuration": "terminus-2; effort xhigh; one attempt; clean 2026-06-25 release", "score": 0.34615384615384615, "raw_score": 0.34615384615384615, "raw_scale": 1, "source_record": "perfect|Claude-Opus-4.8", "passed_tasks": 9, "task_count": 26 }, { "id": "2bfbc4583cf7df2bd7f7", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "terminus-2; thinking adaptive; one attempt; clean 2026-06-25 release", "score": 0.07692307692307693, "raw_score": 0.07692307692307693, "raw_scale": 1, "source_record": "perfect|MiniMax-M3", "passed_tasks": 2, "task_count": 26 }, { "id": "f441967e1d048b1c6573", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 0.038461538461538464, "raw_score": 0.038461538461538464, "raw_scale": 1, "source_record": "perfect|DeepSeek-V4-Pro", "passed_tasks": 1, "task_count": 26 }, { "id": "fe09f2333b528812d150", "model": "GLM-5.2", "organization": "Z AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 0.038461538461538464, "raw_score": 0.038461538461538464, "raw_scale": 1, "source_record": "perfect|GLM-5.2", "passed_tasks": 1, "task_count": 26 }, { "id": "a743e97c7627887aaf88", "model": "DeepSeek-V4-Flash", "organization": "DeepSeek", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "perfect|DeepSeek-V4-Flash", "passed_tasks": 0, "task_count": 26 }, { "id": "86535ec697d640af68c0", "model": "Doubao-Seed-2.1-Pro", "organization": "", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "perfect|Doubao-Seed-2.1-Pro", "passed_tasks": 0, "task_count": 26 }, { "id": "0520ec8dcdf75e2f84a6", "model": "GLM-5.1", "organization": "Z AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "perfect|GLM-5.1", "passed_tasks": 0, "task_count": 26 }, { "id": "d83d37b744efca35acf4", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "terminus-2; effort high; one attempt; clean 2026-06-25 release", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "perfect|GPT-5.5", "passed_tasks": 0, "task_count": 26 }, { "id": "ed17b87a3695712c4555", "model": "Kimi-K2.6", "organization": "Moonshot AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "perfect|Kimi-K2.6", "passed_tasks": 0, "task_count": 26 }, { "id": "393ea0ad69669813a171", "model": "Kimi-K2.7-Code", "organization": "Moonshot AI", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "perfect|Kimi-K2.7-Code", "passed_tasks": 0, "task_count": 26 }, { "id": "61d39c49262986cf9ac4", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "terminus-2; reasoning split; one attempt; clean 2026-06-25 release", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "perfect|MiniMax-M2.7", "passed_tasks": 0, "task_count": 26 }, { "id": "d686de0ee43dd152b06c", "model": "Qwen3.6-Plus", "organization": "Alibaba", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "perfect|Qwen3.6-Plus", "passed_tasks": 0, "task_count": 26 }, { "id": "77f27c2b8480c69586f7", "model": "Qwen3.7-Max", "organization": "Alibaba", "configuration": "terminus-2; thinking on¹; one attempt; clean 2026-06-25 release", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "perfect|Qwen3.7-Max", "passed_tasks": 0, "task_count": 26 } ] }, { "id": "clawbench-v2-lenient", "name": "ClawBench V2 (Lenient)", "group": "Agent 与工具", "unit": "%", "metric": "Lenient judged task success", "source": { "data_url": "https://claw-bench.com/api/leaderboard.json", "sha256": "7daa673365f612c0364b350e45f3f56f35f6d4dfd8f0aa5bcc986979c545a5e7", "retrieved_at": "2026-11-02T05:58:07.581706+00:00", "name": "ClawBench / UniPat AI", "url": "https://claw-bench.com/leaderboard", "selector": "rows_v2; full 130-task rows; reward", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "V2, 130 tasks; intercepted payload plus DeepSeek V4 Pro judgment using the lenient rubric. Harnesses are retained; the published API snapshot is 2026-05-20.", "zh": "V2、130 道任务；拦截提交内容后由 DeepSeek V4 Pro 按宽松量规判定。保留运行框架；官方 API 快照日期为 2026-05-20。" }, "results": [ { "id": "bac52a971ce3fe2b374a", "model": "claude-opus-4-7", "organization": "Anthropic", "configuration": "hermes; V2; 130 tasks; lenient judge rubric; 2026-05-20", "score": 0.4462, "raw_score": 0.4462, "raw_scale": 1, "source_record": "v2|reward|hermes|claude-opus-4-7", "task_count": 130, "matched_tasks": 58, "intercepted_tasks": 71, "source_date": "2026-05-20" }, { "id": "e23e9506fec0120ef6df", "model": "gpt-5.5", "organization": "OpenAI", "configuration": "hermes; V2; 130 tasks; lenient judge rubric; 2026-05-20", "score": 0.3538, "raw_score": 0.3538, "raw_scale": 1, "source_record": "v2|reward|hermes|gpt-5.5", "task_count": 130, "matched_tasks": 46, "intercepted_tasks": 59, "source_date": "2026-05-20" }, { "id": "5420847ebc67a7c513f3", "model": "glm-5.1", "organization": "Z AI", "configuration": "hermes; V2; 130 tasks; lenient judge rubric; 2026-05-20", "score": 0.3462, "raw_score": 0.3462, "raw_scale": 1, "source_record": "v2|reward|hermes|glm-5.1", "task_count": 130, "matched_tasks": 45, "intercepted_tasks": 63, "source_date": "2026-05-20" }, { "id": "922f61d6ffc6b0f9a8c8", "model": "deepseek-v4-pro", "organization": "DeepSeek", "configuration": "hermes; V2; 130 tasks; lenient judge rubric; 2026-05-20", "score": 0.3385, "raw_score": 0.3385, "raw_scale": 1, "source_record": "v2|reward|hermes|deepseek-v4-pro", "task_count": 130, "matched_tasks": 44, "intercepted_tasks": 57, "source_date": "2026-05-20" }, { "id": "9d4ae61e984983f41484", "model": "deepseek-v4-flash:free", "organization": "DeepSeek", "configuration": "hermes; V2; 130 tasks; lenient judge rubric; 2026-05-20", "score": 0.0233, "raw_score": 0.0233, "raw_scale": 1, "source_record": "v2|reward|hermes|deepseek-v4-flash:free", "task_count": 130, "matched_tasks": 3, "intercepted_tasks": 4, "source_date": "2026-05-20" }, { "id": "8221ef2f77b2198ba55b", "model": "z-ai/glm-4.5-air:free", "organization": "", "configuration": "hermes; V2; 130 tasks; lenient judge rubric; 2026-05-20", "score": 0.0231, "raw_score": 0.0231, "raw_scale": 1, "source_record": "v2|reward|hermes|z-ai/glm-4.5-air:free", "task_count": 130, "matched_tasks": 3, "intercepted_tasks": 6, "source_date": "2026-05-20" }, { "id": "b5614771f03b9187ed6a", "model": "minimax-m2.5:free", "organization": "MiniMax", "configuration": "hermes; V2; 130 tasks; lenient judge rubric; 2026-05-20", "score": 0.0154, "raw_score": 0.0154, "raw_scale": 1, "source_record": "v2|reward|hermes|minimax-m2.5:free", "task_count": 130, "matched_tasks": 2, "intercepted_tasks": 3, "source_date": "2026-05-20" }, { "id": "e7bb722fbb0192d16097", "model": "glm-5.1", "organization": "Z AI", "configuration": "openclaw; V2; 130 tasks; lenient judge rubric; 2026-05-20", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "v2|reward|openclaw|glm-5.1", "task_count": 130, "matched_tasks": 0, "intercepted_tasks": 0, "source_date": "2026-05-20" }, { "id": "523e45df479baefa2554", "model": "openrouter-owl-alpha", "organization": "", "configuration": "hermes; V2; 130 tasks; lenient judge rubric; 2026-05-20", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "v2|reward|hermes|openrouter-owl-alpha", "task_count": 130, "matched_tasks": 0, "intercepted_tasks": 19, "source_date": "2026-05-20" } ] }, { "id": "clawbench-v2-strict", "name": "ClawBench V2 (Strict)", "group": "Agent 与工具", "unit": "%", "metric": "Strict judged task success", "source": { "data_url": "https://claw-bench.com/api/leaderboard.json", "sha256": "7daa673365f612c0364b350e45f3f56f35f6d4dfd8f0aa5bcc986979c545a5e7", "retrieved_at": "2026-11-02T05:58:07.581706+00:00", "name": "ClawBench / UniPat AI", "url": "https://claw-bench.com/leaderboard", "selector": "rows_v2; full 130-task rows; reward_strict", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "V2, 130 tasks; intercepted payload plus DeepSeek V4 Pro judgment using the strict rubric. Harnesses are retained; the published API snapshot is 2026-05-20.", "zh": "V2、130 道任务；拦截提交内容后由 DeepSeek V4 Pro 按严格量规判定。保留运行框架；官方 API 快照日期为 2026-05-20。" }, "results": [ { "id": "357d03e0e7b0680e1504", "model": "claude-opus-4-7", "organization": "Anthropic", "configuration": "hermes; V2; 130 tasks; strict judge rubric; 2026-05-20", "score": 0.2462, "raw_score": 0.2462, "raw_scale": 1, "source_record": "v2|reward_strict|hermes|claude-opus-4-7", "task_count": 130, "matched_tasks": 32, "intercepted_tasks": 71, "source_date": "2026-05-20" }, { "id": "dbee25ad25cc4a2f9cc4", "model": "gpt-5.5", "organization": "OpenAI", "configuration": "hermes; V2; 130 tasks; strict judge rubric; 2026-05-20", "score": 0.1846, "raw_score": 0.1846, "raw_scale": 1, "source_record": "v2|reward_strict|hermes|gpt-5.5", "task_count": 130, "matched_tasks": 24, "intercepted_tasks": 59, "source_date": "2026-05-20" }, { "id": "026b696b5a0a558ae0ec", "model": "glm-5.1", "organization": "Z AI", "configuration": "hermes; V2; 130 tasks; strict judge rubric; 2026-05-20", "score": 0.1769, "raw_score": 0.1769, "raw_scale": 1, "source_record": "v2|reward_strict|hermes|glm-5.1", "task_count": 130, "matched_tasks": 23, "intercepted_tasks": 63, "source_date": "2026-05-20" }, { "id": "2af1a4b14a80c581627f", "model": "deepseek-v4-pro", "organization": "DeepSeek", "configuration": "hermes; V2; 130 tasks; strict judge rubric; 2026-05-20", "score": 0.1231, "raw_score": 0.1231, "raw_scale": 1, "source_record": "v2|reward_strict|hermes|deepseek-v4-pro", "task_count": 130, "matched_tasks": 16, "intercepted_tasks": 57, "source_date": "2026-05-20" }, { "id": "9715978cabaabf24e144", "model": "z-ai/glm-4.5-air:free", "organization": "", "configuration": "hermes; V2; 130 tasks; strict judge rubric; 2026-05-20", "score": 0.0077, "raw_score": 0.0077, "raw_scale": 1, "source_record": "v2|reward_strict|hermes|z-ai/glm-4.5-air:free", "task_count": 130, "matched_tasks": 1, "intercepted_tasks": 6, "source_date": "2026-05-20" }, { "id": "9fc55b9afc573c3a2887", "model": "deepseek-v4-flash:free", "organization": "DeepSeek", "configuration": "hermes; V2; 130 tasks; strict judge rubric; 2026-05-20", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "v2|reward_strict|hermes|deepseek-v4-flash:free", "task_count": 130, "matched_tasks": 0, "intercepted_tasks": 4, "source_date": "2026-05-20" }, { "id": "5433503fe14f7e2f6419", "model": "glm-5.1", "organization": "Z AI", "configuration": "openclaw; V2; 130 tasks; strict judge rubric; 2026-05-20", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "v2|reward_strict|openclaw|glm-5.1", "task_count": 130, "matched_tasks": 0, "intercepted_tasks": 0, "source_date": "2026-05-20" }, { "id": "a5f11aad1339125878be", "model": "minimax-m2.5:free", "organization": "MiniMax", "configuration": "hermes; V2; 130 tasks; strict judge rubric; 2026-05-20", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "v2|reward_strict|hermes|minimax-m2.5:free", "task_count": 130, "matched_tasks": 0, "intercepted_tasks": 3, "source_date": "2026-05-20" }, { "id": "9b743a4bb51917c3a185", "model": "openrouter-owl-alpha", "organization": "", "configuration": "hermes; V2; 130 tasks; strict judge rubric; 2026-05-20", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "v2|reward_strict|hermes|openrouter-owl-alpha", "task_count": 130, "matched_tasks": 0, "intercepted_tasks": 19, "source_date": "2026-05-20" } ] }, { "id": "echo-overall", "name": "Echo (Overall)", "group": "预测与决策", "unit": "elo", "metric": "Elo", "source": { "data_url": "https://echo.unipat.ai/api/v2/rankings?category=Overall", "sha256": "a69dae84519601d0783d55487d276c5d6d1e27eb7e361b0b27344f06973a5ad9", "retrieved_at": "2026-11-02T05:58:07.937434+00:00", "name": "UniPat AI", "url": "https://echo.unipat.ai/leaderboard", "selector": "api/v2/rankings?category=Overall: eloScore; provider=official; excludes market baseline", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "Overall prediction Elo from the public rankings API. Market/human baselines are excluded; this is a manual snapshot of a changing ranking.", "zh": "公开榜单 API 的总体预测 Elo；不含市场／人类基线。本页保留本次手动更新的快照。" }, "results": [ { "id": "35135aaa6cc23891b40c", "model": "EchoZ-1.0", "organization": "UniPat AI", "configuration": "Overall; published Elo", "score": 1025.6, "raw_score": 1025.6, "raw_scale": 1, "source_record": "29", "resolved_predictions": 2684, "battles": 93830, "elo_margin": 1.9 }, { "id": "6208ef0f7750c111548c", "model": "Deepseek-V4-Pro", "organization": "DeepSeek", "configuration": "Overall; published Elo", "score": 1018.4, "raw_score": 1018.4, "raw_scale": 1, "source_record": "58", "resolved_predictions": 953, "battles": 47782, "elo_margin": 2.0 }, { "id": "7688b6ef52555962d008", "model": "Kimi-K2.6", "organization": "Moonshot AI", "configuration": "Overall; published Elo", "score": 1017.4, "raw_score": 1017.4, "raw_scale": 1, "source_record": "56", "resolved_predictions": 808, "battles": 17319, "elo_margin": 3.3 }, { "id": "d4735e3a265e16eee03f", "model": "Claude-Opus-4.6", "organization": "Anthropic", "configuration": "Overall; published Elo", "score": 1008.6, "raw_score": 1008.6, "raw_scale": 1, "source_record": "2", "resolved_predictions": 3480, "battles": 139682, "elo_margin": 1.8 }, { "id": "59e19706d51d39f66711", "model": "Gemini-3.1-Pro", "organization": "Google", "configuration": "Overall; published Elo", "score": 1006.3, "raw_score": 1006.3, "raw_scale": 1, "source_record": "28", "resolved_predictions": 2783, "battles": 111297, "elo_margin": 1.9 }, { "id": "c837649cce43f2729138", "model": "GLM-5.1", "organization": "Zhipu AI", "configuration": "Overall; published Elo", "score": 1005.6, "raw_score": 1005.6, "raw_scale": 1, "source_record": "57", "resolved_predictions": 740, "battles": 15762, "elo_margin": 2.9 }, { "id": "4e07408562bedb8b60ce", "model": "Grok-4.1-Fast", "organization": "xAI", "configuration": "Overall; published Elo", "score": 1003.2, "raw_score": 1003.2, "raw_scale": 1, "source_record": "3", "resolved_predictions": 2810, "battles": 120984, "elo_margin": 1.9 }, { "id": "2fca346db656187102ce", "model": "Minimax-M2.7", "organization": "MiniMax", "configuration": "Overall; published Elo", "score": 1001.8, "raw_score": 1001.8, "raw_scale": 1, "source_record": "54", "resolved_predictions": 1004, "battles": 19146, "elo_margin": 3.0 }, { "id": "6b86b273ff34fce19d6b", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Overall; published Elo", "score": 1001.4, "raw_score": 1001.4, "raw_scale": 1, "source_record": "1", "resolved_predictions": 2755, "battles": 115961, "elo_margin": 2.1 }, { "id": "7a61b53701befdae0eee", "model": "Grok-4.2-beta", "organization": "xAI", "configuration": "Overall; published Elo", "score": 1001.3, "raw_score": 1001.3, "raw_scale": 1, "source_record": "37", "resolved_predictions": 1656, "battles": 59014, "elo_margin": 2.6 }, { "id": "eb1e33e8a81b697b7585", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "Overall; published Elo", "score": 999.3, "raw_score": 999.3, "raw_scale": 1, "source_record": "31", "resolved_predictions": 2484, "battles": 83867, "elo_margin": 1.9 }, { "id": "8527a891e224136950ff", "model": "Seed-2.0-Pro", "organization": "ByteDance", "configuration": "Overall; published Elo", "score": 996.2, "raw_score": 996.2, "raw_scale": 1, "source_record": "14", "resolved_predictions": 3313, "battles": 135774, "elo_margin": 1.9 }, { "id": "aea92132c4cbeb263e6a", "model": "Qwen3.5-397B-A17B", "organization": "Alibaba Cloud", "configuration": "Overall; published Elo", "score": 986.0, "raw_score": 986.0, "raw_scale": 1, "source_record": "38", "resolved_predictions": 2050, "battles": 60042, "elo_margin": 2.5 }, { "id": "4a44dc15364204a80fe8", "model": "GLM-5", "organization": "Zhipu AI", "configuration": "Overall; published Elo", "score": 983.9, "raw_score": 983.9, "raw_scale": 1, "source_record": "10", "resolved_predictions": 2876, "battles": 115961, "elo_margin": 2.1 }, { "id": "7902699be42c8a8e46fb", "model": "Minimax-M2.5", "organization": "MiniMax", "configuration": "Overall; published Elo", "score": 979.7, "raw_score": 979.7, "raw_scale": 1, "source_record": "7", "resolved_predictions": 2805, "battles": 121322, "elo_margin": 2.1 }, { "id": "2c624232cdd221771294", "model": "Qwen3-Max", "organization": "Alibaba Cloud", "configuration": "Overall; published Elo", "score": 966.5, "raw_score": 966.5, "raw_scale": 1, "source_record": "8", "resolved_predictions": 2509, "battles": 111409, "elo_margin": 2.2 }, { "id": "19581e27de7ced00ff1c", "model": "Deepseek-V3.2", "organization": "DeepSeek", "configuration": "Overall; published Elo", "score": 964.3, "raw_score": 964.3, "raw_scale": 1, "source_record": "9", "resolved_predictions": 3324, "battles": 132601, "elo_margin": 1.9 }, { "id": "ef2d127de37b942baad0", "model": "Kimi-K2.5", "organization": "Moonshot AI", "configuration": "Overall; published Elo", "score": 920.2, "raw_score": 920.2, "raw_scale": 1, "source_record": "5", "resolved_predictions": 2934, "battles": 121566, "elo_margin": 2.1 } ] }, { "id": "babyvision", "name": "BabyVision", "group": "视觉与视频", "unit": "%", "metric": "Accuracy", "source": { "data_url": "https://unipat.ai/benchmarks/BabyVision", "sha256": "9c09986ee445c9877375abf4d2a18623827c8a30a6d0f16264967aa3071d9d1a", "retrieved_at": "2026-11-02T05:58:00.625095+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/BabyVision", "selector": "BabyVision Leaderboard: Score; exclude Type=Human; preserve tool labels", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://arxiv.org/html/2601.06521v2", "sha256": "297cbfef328d3b48647589d2de1231ad5d0e15b8e1daa0cb27f2e1dc22aa81c9", "retrieved_at": "2026-11-02T05:59:46.651227+00:00" } ] }, "protocol": { "en": "Published visual-reasoning scores retain their tool and reasoning settings. Appendix Table 6 supplies additional tool-free Qwen3VL results.", "zh": "官方视觉推理成绩，保留工具与推理设置。论文附录表 6 补充了 Qwen3VL 的无工具成绩。" }, "results": [ { "id": "198b025aca6542de9b9d", "model": "Qwen3.8-Max (with tools)", "organization": "Alibaba", "configuration": "Open", "score": 0.9129999999999999, "raw_score": 91.3, "raw_scale": 100, "source_record": "Qwen3.8-Max (with tools)", "source_date": "Aug 3, 2026" }, { "id": "d98b2be776ddd89cb36e", "model": "Fable 5 (max) (with tools)", "organization": "Anthropic", "configuration": "Proprietary", "score": 0.905, "raw_score": 90.5, "raw_scale": 100, "source_record": "Fable 5 (max) (with tools)", "source_date": "Jun 9, 2026" }, { "id": "25bd2839f41a471d257d", "model": "GPT-5.6 Sol (max) (with tools)", "organization": "OpenAI", "configuration": "Proprietary", "score": 0.889, "raw_score": 88.9, "raw_scale": 100, "source_record": "GPT-5.6 Sol (max) (with tools)", "source_date": "Jun 26, 2026" }, { "id": "9d541aca072a74f6067b", "model": "Kimi K3 (with Python tools)", "organization": "Moonshot AI", "configuration": "Open", "score": 0.857, "raw_score": 85.7, "raw_scale": 100, "source_record": "Kimi K3 (with Python tools)", "source_date": "Jul 14, 2026" }, { "id": "c6cd0a3e595f13f12215", "model": "GPT-5.5 (xhigh) (with tools)", "organization": "OpenAI", "configuration": "Proprietary", "score": 0.836, "raw_score": 83.6, "raw_scale": 100, "source_record": "GPT-5.5 (xhigh) (with tools)", "source_date": "Apr 23, 2026" }, { "id": "ebfb011f410de5a2c765", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "Open", "score": 0.82, "raw_score": 82.0, "raw_scale": 100, "source_record": "Qwen3.8-Max", "source_date": "Aug 3, 2026" }, { "id": "fe22f7d04a585d38287f", "model": "Claude-Opus-4.8 (max) (with tools)", "organization": "Anthropic", "configuration": "Proprietary", "score": 0.812, "raw_score": 81.2, "raw_scale": 100, "source_record": "Claude-Opus-4.8 (max) (with tools)", "source_date": "May 28, 2026" }, { "id": "68611f40bbe481bdd6e3", "model": "Muse Spark 1.1 (with tools)", "organization": "Meta", "configuration": "Proprietary", "score": 0.763, "raw_score": 76.3, "raw_scale": 100, "source_record": "Muse Spark 1.1 (with tools)", "source_date": "Jul 9, 2026" }, { "id": "77ecc2b98899c95ae295", "model": "Seed-2.1-Pro", "organization": "ByteDance", "configuration": "Proprietary", "score": 0.737, "raw_score": 73.7, "raw_scale": 100, "source_record": "Seed-2.1-Pro", "source_date": "Jun 23, 2026" }, { "id": "e6d72991efe147b270c5", "model": "Qwen3.7-Plus (with tools)", "organization": "Alibaba", "configuration": "Proprietary", "score": 0.7040000000000001, "raw_score": 70.4, "raw_scale": 100, "source_record": "Qwen3.7-Plus (with tools)", "source_date": "Jun 2, 2026" }, { "id": "1b449041a270b17599be", "model": "Kimi-K2.6 (with tools)", "organization": "Moonshot AI", "configuration": "Open", "score": 0.685, "raw_score": 68.5, "raw_scale": 100, "source_record": "Kimi-K2.6 (with tools)", "source_date": "Apr 21, 2026" }, { "id": "a247f7d209c8be7289ca", "model": "Seed-2.0-Pro (high)", "organization": "ByteDance", "configuration": "Proprietary", "score": 0.606, "raw_score": 60.6, "raw_scale": 100, "source_record": "Seed-2.0-Pro (high)", "source_date": "Feb 14, 2026" }, { "id": "18c7c5baac450bdaf2a0", "model": "Gemini-3.1-Pro", "organization": "Google", "configuration": "Proprietary", "score": 0.516, "raw_score": 51.6, "raw_scale": 100, "source_record": "Gemini-3.1-Pro", "source_date": "Feb 23, 2026" }, { "id": "4e59fffea08ce34a18ba", "model": "GPT-5.4 (xhigh)", "organization": "OpenAI", "configuration": "Proprietary", "score": 0.49700000000000005, "raw_score": 49.7, "raw_scale": 100, "source_record": "GPT-5.4 (xhigh)", "source_date": "Mar 5, 2026" }, { "id": "b6c41da452822ed745c8", "model": "Gemini-3.0-Pro", "organization": "Google", "configuration": "Proprietary", "score": 0.49700000000000005, "raw_score": 49.7, "raw_scale": 100, "source_record": "Gemini-3.0-Pro", "source_date": "Nov 18, 2025" }, { "id": "9ef5351c75513ba35fa9", "model": "Qwen3.5-397B-A17B", "organization": "Alibaba", "configuration": "Open", "score": 0.433, "raw_score": 43.3, "raw_scale": 100, "source_record": "Qwen3.5-397B-A17B", "source_date": "Feb 16, 2026" }, { "id": "f8f198fcbd0d4bdd1a3c", "model": "Kimi-K2.5", "organization": "Moonshot AI", "configuration": "Open", "score": 0.365, "raw_score": 36.5, "raw_scale": 100, "source_record": "Kimi-K2.5", "source_date": "Jan 27, 2026" }, { "id": "4dda9489c142ee68404e", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Proprietary", "score": 0.344, "raw_score": 34.4, "raw_scale": 100, "source_record": "GPT-5.2", "source_date": "Dec 11, 2025" }, { "id": "f299175a2fe3972e8303", "model": "Seed-1.8", "organization": "ByteDance", "configuration": "Proprietary", "score": 0.302, "raw_score": 30.2, "raw_scale": 100, "source_record": "Seed-1.8", "source_date": "Dec 18, 2025" }, { "id": "0d819249c80a084d8b8a", "model": "Qwen3VL-235B-Thinking", "organization": "Alibaba", "configuration": "Open", "score": 0.222, "raw_score": 22.2, "raw_scale": 100, "source_record": "Qwen3VL-235B-Thinking", "source_date": "Sep 23, 2025" }, { "id": "4f26899d1c6b9da688f1", "model": "Qwen3VL-235B-Instruct", "organization": "Alibaba", "configuration": "Tool-free; three runs; 388 items", "score": 0.195, "raw_score": 19.5, "raw_scale": 100, "source_record": "Table 6|Overall|Qwen3VL-235B-Instruct", "raw_std": 1.4, "task_count": 388, "report_url": "https://arxiv.org/html/2601.06521v2#A3.T6" }, { "id": "fe08e2204c98aed74cee", "model": "InternVL3.5-241B", "organization": "OpenGVLab", "configuration": "Open", "score": 0.192, "raw_score": 19.2, "raw_scale": 100, "source_record": "InternVL3.5-241B", "source_date": "Aug 26, 2025" }, { "id": "eb0c498354b79c29d17c", "model": "Qwen3-VL-Plus", "organization": "Alibaba", "configuration": "Proprietary", "score": 0.192, "raw_score": 19.2, "raw_scale": 100, "source_record": "Qwen3-VL-Plus", "source_date": "Sep 23, 2025" }, { "id": "fc7109e61d4b5cecafff", "model": "GLM4.6V", "organization": "Zhipu AI", "configuration": "Open", "score": 0.17600000000000002, "raw_score": 17.6, "raw_scale": 100, "source_record": "GLM4.6V", "source_date": "Dec 8, 2025" }, { "id": "50024d53fec1d402ccaf", "model": "Qwen3VL-32B-Thinking", "organization": "Alibaba", "configuration": "Tool-free; three runs; 388 items", "score": 0.174, "raw_score": 17.4, "raw_scale": 100, "source_record": "Table 6|Overall|Qwen3VL-32B-Thinking", "raw_std": 2.0, "task_count": 388, "report_url": "https://arxiv.org/html/2601.06521v2#A3.T6" }, { "id": "87704efa4c8acbe57990", "model": "Grok-4", "organization": "xAI", "configuration": "Proprietary", "score": 0.162, "raw_score": 16.2, "raw_scale": 100, "source_record": "Grok-4", "source_date": "Jul 9, 2025" }, { "id": "dde8a937c5a6f204fef0", "model": "MimoVL-7B-RL", "organization": "Xiaomi", "configuration": "Open", "score": 0.151, "raw_score": 15.1, "raw_scale": 100, "source_record": "MimoVL-7B-RL", "source_date": "Apr 23, 2025" }, { "id": "8f39cef88f589aab319f", "model": "Claude-Opus-4.6", "organization": "Anthropic", "configuration": "Proprietary", "score": 0.14800000000000002, "raw_score": 14.8, "raw_scale": 100, "source_record": "Claude-Opus-4.6", "source_date": "Feb 5, 2026" }, { "id": "b107af1cf916d12387e5", "model": "Step3", "organization": "StepFun", "configuration": "Open", "score": 0.147, "raw_score": 14.7, "raw_scale": 100, "source_record": "Step3", "source_date": "Jul 31, 2025" }, { "id": "45129c408ceb8916bcfa", "model": "Qwen3VL-4B-Thinking", "organization": "Alibaba", "configuration": "Tool-free; three runs; 388 items", "score": 0.146, "raw_score": 14.6, "raw_scale": 100, "source_record": "Table 6|Overall|Qwen3VL-4B-Thinking", "raw_std": 1.2, "task_count": 388, "report_url": "https://arxiv.org/html/2601.06521v2#A3.T6" }, { "id": "4eff24d1709427cf0db3", "model": "Claude-4.5-Opus", "organization": "Anthropic", "configuration": "Proprietary", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "Claude-4.5-Opus", "source_date": "Nov 24, 2025" }, { "id": "6735c775add592643a8d", "model": "Qwen3VL-8B-Thinking", "organization": "Alibaba", "configuration": "Tool-free; three runs; 388 items", "score": 0.131, "raw_score": 13.1, "raw_scale": 100, "source_record": "Table 6|Overall|Qwen3VL-8B-Thinking", "raw_std": 1.1, "task_count": 388, "report_url": "https://arxiv.org/html/2601.06521v2#A3.T6" }, { "id": "96bdcd4e393800a62adc", "model": "KimiVL-A3B", "organization": "Moonshot AI", "configuration": "Open", "score": 0.124, "raw_score": 12.4, "raw_scale": 100, "source_record": "KimiVL-A3B", "source_date": "Apr 10, 2025" } ] }, { "id": "sciencevision-3d", "name": "ScienceVision (3D)", "group": "数学与科学", "unit": "score", "metric": "Mean answer score", "source": { "data_url": "https://unipat.ai/static/sciencevision/data/benchmark.js?v=e77dc041", "sha256": "e77dc041c85ce6db43d89a0dd0882d2f7035c4409d5c965b287ebba38131302f", "retrieved_at": "2026-11-02T05:58:07.673852+00:00", "name": "UniPat AI", "url": "https://unipat.ai/blog/ScienceVision", "selector": "SV_BENCHMARK.leaderboard.models[].settings[cc-3d].mean; 250 questions", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://unipat.ai/blog/ScienceVision", "sha256": "122455e0ac78abe066aff14488fa986fa688751af3eab91fb2ae2090b39f5da5", "retrieved_at": "2026-11-02T05:58:02.447980+00:00" } ] }, "protocol": { "en": "3D; mean answer-component score on 0–1 over 250 questions, up to three runs; ineligible trials count as zero. GPT rows use Codex, others Claude Code; source segments span different runs.", "zh": "3D 源文件；250 道题、最多三次运行的回答分量平均分，保留 0–1 原单位，无效试验计零。GPT 使用 Codex，其他模型使用 Claude Code；来源拼接不同运行批次。" }, "results": [ { "id": "18f23ab94e45fca84f2f", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Codex; cc-3d; up to 3 runs; 250 questions", "score": 0.5291, "raw_score": 0.5291, "raw_scale": 1, "source_record": "leaderboard.models|gpt-6-astra|cc-3d|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "a07752bbf5e507b34dd2", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Codex; cc-3d; up to 3 runs; 250 questions", "score": 0.4893, "raw_score": 0.4893, "raw_scale": 1, "source_record": "leaderboard.models|gpt-6.1-sol|cc-3d|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "dfccb42a68f65a87cfaa", "model": "Claude Fable 5.1", "organization": "Anthropic", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.4619, "raw_score": 0.4619, "raw_scale": 1, "source_record": "leaderboard.models|claude-fable-5-1|cc-3d|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "95b31c8f90d0760d1026", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.4539, "raw_score": 0.4539, "raw_scale": 1, "source_record": "leaderboard.models|claude-opus-5-5|cc-3d|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "3a96e02bf08fe4a9c63c", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.4278, "raw_score": 0.4278, "raw_scale": 1, "source_record": "leaderboard.models|gemini-4-argon|cc-3d|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "074088566f08ac63f9ee", "model": "Qwen3.8 Max 0902", "organization": "Alibaba", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.383, "raw_score": 0.383, "raw_scale": 1, "source_record": "leaderboard.models|qwen3.8-max-0902|cc-3d|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "15d36c2bb639d8ce9a0c", "model": "Qwen3.8 Flash", "organization": "Alibaba", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.3616, "raw_score": 0.3616, "raw_scale": 1, "source_record": "leaderboard.models|qwen3.8-flash|cc-3d|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "33750ed8e34e22a19fee", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.3609, "raw_score": 0.3609, "raw_scale": 1, "source_record": "leaderboard.models|kimi-k3|cc-3d|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "5568664c6cd495388444", "model": "GLM-5.3 Flash", "organization": "Zhipu AI", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.2846, "raw_score": 0.2846, "raw_scale": 1, "source_record": "leaderboard.models|glm-5.3-flash|cc-3d|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "9e429e0b451a8b336975", "model": "DeepSeek V4 Flash", "organization": "DeepSeek", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.2652, "raw_score": 0.2652, "raw_scale": 1, "source_record": "leaderboard.models|deepseek-v4-flash|cc-3d|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "37decacb059abb72d2e3", "model": "Doubao Seed Evolving", "organization": "ByteDance", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.2435, "raw_score": 0.2435, "raw_scale": 1, "source_record": "leaderboard.models|doubao-seed-evolving|cc-3d|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "032354efa5acd38d5472", "model": "Hunyuan-4 Preview", "organization": "Tencent", "configuration": "Claude Code; cc-3d; up to 3 runs; 250 questions", "score": 0.1598, "raw_score": 0.1598, "raw_scale": 1, "source_record": "leaderboard.models|hy4-preview|cc-3d|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } } ], "score_max": 1 }, { "id": "sciencevision-mcp", "name": "ScienceVision (3D + MCP)", "group": "数学与科学", "unit": "score", "metric": "Mean answer score", "source": { "data_url": "https://unipat.ai/static/sciencevision/data/benchmark.js?v=e77dc041", "sha256": "e77dc041c85ce6db43d89a0dd0882d2f7035c4409d5c965b287ebba38131302f", "retrieved_at": "2026-11-02T05:58:07.673852+00:00", "name": "UniPat AI", "url": "https://unipat.ai/blog/ScienceVision", "selector": "SV_BENCHMARK.leaderboard.models[].settings[cc-mcp].mean; 250 questions", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://unipat.ai/blog/ScienceVision", "sha256": "122455e0ac78abe066aff14488fa986fa688751af3eab91fb2ae2090b39f5da5", "retrieved_at": "2026-11-02T05:58:02.447980+00:00" } ] }, "protocol": { "en": "3D + MCP; mean answer-component score on 0–1 over 250 questions, up to three runs; ineligible trials count as zero. GPT rows use Codex, others Claude Code; source segments span different runs.", "zh": "3D 源文件与 MCP 检视工具；250 道题、最多三次运行的回答分量平均分，保留 0–1 原单位，无效试验计零。GPT 使用 Codex，其他模型使用 Claude Code；来源拼接不同运行批次。" }, "results": [ { "id": "f32052217e0ef35f92cf", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Codex; cc-mcp; up to 3 runs; 250 questions", "score": 0.5282, "raw_score": 0.5282, "raw_scale": 1, "source_record": "leaderboard.models|gpt-6-astra|cc-mcp|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "e147d9a7d8b1a4df6771", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Codex; cc-mcp; up to 3 runs; 250 questions", "score": 0.511, "raw_score": 0.511, "raw_scale": 1, "source_record": "leaderboard.models|gpt-6.1-sol|cc-mcp|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "24af5eadbfffbb3a1c2d", "model": "Claude Fable 5.1", "organization": "Anthropic", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.4631, "raw_score": 0.4631, "raw_scale": 1, "source_record": "leaderboard.models|claude-fable-5-1|cc-mcp|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "d1f791df14636cc5e3e3", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.4343, "raw_score": 0.4343, "raw_scale": 1, "source_record": "leaderboard.models|claude-opus-5-5|cc-mcp|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "2da6ff6fe71ed15e8908", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.4321, "raw_score": 0.4321, "raw_scale": 1, "source_record": "leaderboard.models|gemini-4-argon|cc-mcp|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "188b2d388d17cda3d232", "model": "Qwen3.8 Max 0902", "organization": "Alibaba", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.3736, "raw_score": 0.3736, "raw_scale": 1, "source_record": "leaderboard.models|qwen3.8-max-0902|cc-mcp|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "ea1c8b8be110a585cdba", "model": "Qwen3.8 Flash", "organization": "Alibaba", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.3681, "raw_score": 0.3681, "raw_scale": 1, "source_record": "leaderboard.models|qwen3.8-flash|cc-mcp|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "7c0bb246e94675ec42a2", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.3369, "raw_score": 0.3369, "raw_scale": 1, "source_record": "leaderboard.models|kimi-k3|cc-mcp|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "991f2a9095c48e23fce4", "model": "GLM-5.3 Flash", "organization": "Zhipu AI", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.2858, "raw_score": 0.2858, "raw_scale": 1, "source_record": "leaderboard.models|glm-5.3-flash|cc-mcp|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "343ed7e55e6c1d763732", "model": "DeepSeek V4 Flash", "organization": "DeepSeek", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.2678, "raw_score": 0.2678, "raw_scale": 1, "source_record": "leaderboard.models|deepseek-v4-flash|cc-mcp|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "699d6ccd0b1b9fc5ac43", "model": "Doubao Seed Evolving", "organization": "ByteDance", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.2527, "raw_score": 0.2527, "raw_scale": 1, "source_record": "leaderboard.models|doubao-seed-evolving|cc-mcp|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "7d775b22c2f34b536810", "model": "Hunyuan-4 Preview", "organization": "Tencent", "configuration": "Claude Code; cc-mcp; up to 3 runs; 250 questions", "score": 0.1738, "raw_score": 0.1738, "raw_scale": 1, "source_record": "leaderboard.models|hy4-preview|cc-mcp|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } } ], "score_max": 1 }, { "id": "sciencevision-2d-6", "name": "ScienceVision (2D, six views)", "group": "数学与科学", "unit": "score", "metric": "Mean answer score", "source": { "data_url": "https://unipat.ai/static/sciencevision/data/benchmark.js?v=e77dc041", "sha256": "e77dc041c85ce6db43d89a0dd0882d2f7035c4409d5c965b287ebba38131302f", "retrieved_at": "2026-11-02T05:58:07.673852+00:00", "name": "UniPat AI", "url": "https://unipat.ai/blog/ScienceVision", "selector": "SV_BENCHMARK.leaderboard.models[].settings[cc-2d-6].mean; 250 questions", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://unipat.ai/blog/ScienceVision", "sha256": "122455e0ac78abe066aff14488fa986fa688751af3eab91fb2ae2090b39f5da5", "retrieved_at": "2026-11-02T05:58:02.447980+00:00" } ] }, "protocol": { "en": "2D, six views; mean answer-component score on 0–1 over 250 questions, up to three runs; ineligible trials count as zero. GPT rows use Codex, others Claude Code; source segments span different runs.", "zh": "六幅 2D 固定视图；250 道题、最多三次运行的回答分量平均分，保留 0–1 原单位，无效试验计零。GPT 使用 Codex，其他模型使用 Claude Code；来源拼接不同运行批次。" }, "results": [ { "id": "72b4107ad8f8c5c18739", "model": "Claude Fable 5.1", "organization": "Anthropic", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.2965, "raw_score": 0.2965, "raw_scale": 1, "source_record": "leaderboard.models|claude-fable-5-1|cc-2d-6|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "35ba7b4b4049e7276963", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Codex; cc-2d-6; up to 3 runs; 250 questions", "score": 0.2498, "raw_score": 0.2498, "raw_scale": 1, "source_record": "leaderboard.models|gpt-6-astra|cc-2d-6|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "8183556b8ac9106aa4f9", "model": "Qwen3.8 Flash", "organization": "Alibaba", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.2468, "raw_score": 0.2468, "raw_scale": 1, "source_record": "leaderboard.models|qwen3.8-flash|cc-2d-6|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "59cd73914e84dd0c1d86", "model": "Qwen3.8 Max 0902", "organization": "Alibaba", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.245, "raw_score": 0.245, "raw_scale": 1, "source_record": "leaderboard.models|qwen3.8-max-0902|cc-2d-6|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "2f5807c63af7b2874328", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.2383, "raw_score": 0.2383, "raw_scale": 1, "source_record": "leaderboard.models|claude-opus-5-5|cc-2d-6|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "8559155f4317fe1803c1", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.2148, "raw_score": 0.2148, "raw_scale": 1, "source_record": "leaderboard.models|kimi-k3|cc-2d-6|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "0dc934769f4b27a456aa", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Codex; cc-2d-6; up to 3 runs; 250 questions", "score": 0.2044, "raw_score": 0.2044, "raw_scale": 1, "source_record": "leaderboard.models|gpt-6.1-sol|cc-2d-6|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "81d28ff9ccbb134c3625", "model": "DeepSeek V4 Flash", "organization": "DeepSeek", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.1661, "raw_score": 0.1661, "raw_scale": 1, "source_record": "leaderboard.models|deepseek-v4-flash|cc-2d-6|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "f4e0c5a625d3804e156c", "model": "GLM-5.3 Flash", "organization": "Zhipu AI", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.1521, "raw_score": 0.1521, "raw_scale": 1, "source_record": "leaderboard.models|glm-5.3-flash|cc-2d-6|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "fd80ce79f01573799b73", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.1462, "raw_score": 0.1462, "raw_scale": 1, "source_record": "leaderboard.models|gemini-4-argon|cc-2d-6|mean", "source_panel": "cross-source", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "708dab8372bea1d40011", "model": "Doubao Seed Evolving", "organization": "ByteDance", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.1102, "raw_score": 0.1102, "raw_scale": 1, "source_record": "leaderboard.models|doubao-seed-evolving|cc-2d-6|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } }, { "id": "08b203da90cc5cfa546f", "model": "Hunyuan-4 Preview", "organization": "Tencent", "configuration": "Claude Code; cc-2d-6; up to 3 runs; 250 questions", "score": 0.0824, "raw_score": 0.0824, "raw_scale": 1, "source_record": "leaderboard.models|hy4-preview|cc-2d-6|mean", "source_panel": "same-harness", "reported_runs": 3, "run_metadata": { "set": "ScienceVision", "generated": "2026-10-21", "sha256": "8d7180f845de676df410c1ac1e5b413a3e37feaa31455c8e4ca71424ad3c0569", "viewDigest": "b5fcc3d5d667dff2d7bbbe3036e6265a54e9af160b16428b6539771bd9531843" } } ], "score_max": 1 }, { "id": "deepterminalbench", "name": "DeepTerminalBench (Terminal-X)", "group": "编程", "unit": "%", "metric": "Pass@1", "source": { "data_url": "https://unipat.ai/blog/TerminalX", "sha256": "644ee3f71555674209d25d2f0ee943195668a07bb89e70e07e936d1c3cd7b752", "retrieved_at": "2026-11-02T05:58:10.153044+00:00", "name": "UniPat AI", "url": "https://unipat.ai/blog/TerminalX", "selector": "DeepTerminalBench main-results table: Pass@1; 50 tasks; exclude sibling EvoCode/Roadmap legacy tables", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "50 terminal tasks; terminus-2, one attempt. Passing requires hidden verification reward = 1; the separate pass@4 calibration is not the ranking score.", "zh": "50 道终端任务；terminus-2、单次尝试。隐藏验证奖励为 1 才算通过；不使用任务校准的 pass@4 作为排名成绩。" }, "results": [ { "id": "790ded0f8078119d4185", "model": "Claude-Opus-4.7", "organization": "Anthropic", "configuration": "terminus-2; one attempt; 50 tasks", "score": 0.34, "raw_score": 34.0, "raw_scale": 100, "source_record": "Claude-Opus-4.7" }, { "id": "9db5201cc1f8dc047be2", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "terminus-2; one attempt; 50 tasks", "score": 0.32, "raw_score": 32.0, "raw_scale": 100, "source_record": "DeepSeek-V4-Pro" }, { "id": "8b8d20f70b4259abba4e", "model": "GPT-5.5-high", "organization": "OpenAI", "configuration": "terminus-2; one attempt; 50 tasks", "score": 0.32, "raw_score": 32.0, "raw_scale": 100, "source_record": "GPT-5.5-high" }, { "id": "e6f6c7f1753c092f3b11", "model": "Kimi-K2.6", "organization": "Moonshot AI", "configuration": "terminus-2; one attempt; 50 tasks", "score": 0.32, "raw_score": 32.0, "raw_scale": 100, "source_record": "Kimi-K2.6" }, { "id": "ff854fc290cb45995ecf", "model": "Qwen-3.6-Plus", "organization": "Alibaba", "configuration": "terminus-2; one attempt; 50 tasks", "score": 0.32, "raw_score": 32.0, "raw_scale": 100, "source_record": "Qwen-3.6-Plus" }, { "id": "539e801876c705582368", "model": "GLM-5.1", "organization": "Z AI", "configuration": "terminus-2; one attempt; 50 tasks", "score": 0.3, "raw_score": 30.0, "raw_scale": 100, "source_record": "GLM-5.1" }, { "id": "fea2d0c5a1f67b24b779", "model": "Qwen-3.5-397B-A17B", "organization": "Alibaba", "configuration": "terminus-2; one attempt; 50 tasks", "score": 0.24, "raw_score": 24.0, "raw_scale": 100, "source_record": "Qwen-3.5-397B-A17B" }, { "id": "18c7c5baac450bdaf2a0", "model": "Gemini-3.1-Pro", "organization": "Google", "configuration": "terminus-2; one attempt; 50 tasks", "score": 0.16, "raw_score": 16.0, "raw_scale": 100, "source_record": "Gemini-3.1-Pro" }, { "id": "ea0d72ff356988b6b5e8", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "terminus-2; one attempt; 50 tasks", "score": 0.1, "raw_score": 10.0, "raw_scale": 100, "source_record": "MiniMax-M2.7" } ] }, { "id": "vibe-coding-arena", "name": "Vibe-Coding Arena (14-task panel)", "group": "编程", "unit": "score", "metric": "Mean process grade", "source": { "data_url": "https://unipat.ai/blog/Vibe-Coding-Arena", "sha256": "d993ac3e2a0c6a911617889bc9bd6595dd04bd8cedd79e23a5a5667816be52f8", "retrieved_at": "2026-11-02T05:58:10.322537+00:00", "name": "UniPat AI", "url": "https://unipat.ai/blog/Vibe-Coding-Arena", "selector": "Per-task scores (all 14): mean of each anonymous A–D column, rounded to 2 decimals as in published summary", "kind": "official_leaderboard", "status": "current_snapshot" }, "protocol": { "en": "Four anonymous models, the same 14 tasks. Human process grades on the original 0–5 scale; identities are undisclosed. This panel is distinct from the larger task inventory.", "zh": "四个匿名模型在相同的 14 道任务上，由人类评定过程表现；保留 0–5 原量纲。模型身份未公开，此面板不等于完整任务库。" }, "results": [ { "id": "82b10417e2c8a2d88a98", "model": "Anonymous model A", "organization": "", "configuration": "Anonymous A–D panel; 14 tasks; human process grading", "score": 4.21, "raw_score": 4.21, "raw_scale": 1, "source_record": "14-task panel|A", "per_task_scores": [ 3.0, 3.0, 5.0, 5.0, 4.0, 5.0, 5.0, 4.0, 4.0, 5.0, 4.0, 5.0, 4.0, 3.0 ] }, { "id": "1d0547fc7659008d15d1", "model": "Anonymous model D", "organization": "", "configuration": "Anonymous A–D panel; 14 tasks; human process grading", "score": 3.57, "raw_score": 3.57, "raw_scale": 1, "source_record": "14-task panel|D", "per_task_scores": [ 4.0, 3.0, 4.0, 3.0, 4.0, 5.0, 4.0, 2.0, 4.0, 4.0, 4.0, 4.0, 2.0, 3.0 ] }, { "id": "7599156458c78d246bb3", "model": "Anonymous model B", "organization": "", "configuration": "Anonymous A–D panel; 14 tasks; human process grading", "score": 2.93, "raw_score": 2.93, "raw_scale": 1, "source_record": "14-task panel|B", "per_task_scores": [ 3.0, 3.0, 3.0, 4.0, 2.0, 4.0, 3.0, 1.0, 3.0, 3.0, 4.0, 3.0, 2.0, 3.0 ] }, { "id": "7809cdc80a7d381bef6b", "model": "Anonymous model C", "organization": "", "configuration": "Anonymous A–D panel; 14 tasks; human process grading", "score": 1.79, "raw_score": 1.79, "raw_scale": 1, "source_record": "14-task panel|C", "per_task_scores": [ 2.0, 2.0, 2.0, 1.0, 1.0, 1.0, 2.0, 1.0, 2.0, 3.0, 3.0, 2.0, 1.0, 2.0 ] } ], "score_max": 5 }, { "id": "surge-riemann", "name": "Riemann-bench", "group": "数学与科学", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Published research-mathematics accuracy; reasoning settings are retained per result.", "zh": "官方研究级数学正确率；逐条保留推理设置。" }, "source": { "name": "Surge AI", "benchmark_creator": "Surge AI", "url": "https://surgehq.ai/benchmarks/riemann-bench", "data_url": "https://surgehq.ai/benchmarks/riemann-bench", "sha256": "3fcc838de91500db5c7b7ffdf3772a985e4d1bb014798fd9516158555e03d1f8", "retrieved_at": "2026-11-02T03:35:08.199671+00:00", "selector": "[data-leaderboard-item]: model label + numeric [data-score]", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "9353e461a4affb00169c", "model": "GPT 5.6 Sol (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.7440000000000001, "raw_score": 74.4, "raw_scale": 100, "source_record": "GPT 5.6 Sol (Max reasoning)|" }, { "id": "f6cc94ca3090a4a25f2a", "model": "GPT 6 Astra (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "GPT 6 Astra (Max reasoning)|" }, { "id": "58e21596a78517c854ca", "model": "Claude Opus 5.5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.696, "raw_score": 69.6, "raw_scale": 100, "source_record": "Claude Opus 5.5 (Adaptive/Max)|" }, { "id": "9db3a02f01736ec8d028", "model": "Claude Opus 5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.68, "raw_score": 68.0, "raw_scale": 100, "source_record": "Claude Opus 5 (Adaptive/Max)|" }, { "id": "f946a767b9d4ee5a5d63", "model": "GPT 5.6 Terra (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.664, "raw_score": 66.4, "raw_scale": 100, "source_record": "GPT 5.6 Terra (Max reasoning)|" }, { "id": "1830705f32b90c041a16", "model": "Claude Fable 5.1 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.6559999999999999, "raw_score": 65.6, "raw_scale": 100, "source_record": "Claude Fable 5.1 (Adaptive/Max)|" }, { "id": "e058db180145cdcf48e3", "model": "GPT 6 Sol (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.648, "raw_score": 64.8, "raw_scale": 100, "source_record": "GPT 6 Sol (Max reasoning)|" }, { "id": "3fdad0139a57069b2d27", "model": "Claude Fable 5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.6, "raw_score": 60.0, "raw_scale": 100, "source_record": "Claude Fable 5 (Adaptive/Max)|" }, { "id": "11202fdc5a0079debdd4", "model": "GPT 6 Luna (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.5920000000000001, "raw_score": 59.2, "raw_scale": 100, "source_record": "GPT 6 Luna (Max reasoning)|" }, { "id": "6349b7a7be0d58473df5", "model": "GPT 5.5 (xHigh reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.552, "raw_score": 55.2, "raw_scale": 100, "source_record": "GPT 5.5 (xHigh reasoning)|" }, { "id": "0f9f52222aa90b209a59", "model": "Gemini 3.8 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 0.512, "raw_score": 51.2, "raw_scale": 100, "source_record": "Gemini 3.8 Flash (High reasoning)|" }, { "id": "8f8ce8676556b80fa20e", "model": "Claude Opus 4.8 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.47200000000000003, "raw_score": 47.2, "raw_scale": 100, "source_record": "Claude Opus 4.8 (Adaptive/Max)|" }, { "id": "ef7067e293c9801323de", "model": "GPT 5.4 (xHigh reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.41600000000000004, "raw_score": 41.6, "raw_scale": 100, "source_record": "GPT 5.4 (xHigh reasoning)|" }, { "id": "2b494471814b49a7c94b", "model": "Gemini 3.7 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 0.392, "raw_score": 39.2, "raw_scale": 100, "source_record": "Gemini 3.7 Flash (High reasoning)|" }, { "id": "d22e89e76852891376da", "model": "DeepSeek V4 Pro (Max reasoning)", "organization": "DeepSeek", "configuration": "", "score": 0.384, "raw_score": 38.4, "raw_scale": 100, "source_record": "DeepSeek V4 Pro (Max reasoning)|" }, { "id": "81896f03145bc246072e", "model": "Grok 4.5 (High reasoning)", "organization": "Grok", "configuration": "", "score": 0.384, "raw_score": 38.4, "raw_scale": 100, "source_record": "Grok 4.5 (High reasoning)|" }, { "id": "188c7f4fc3e592a8f02c", "model": "GPT 5.2 (xHigh reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.376, "raw_score": 37.6, "raw_scale": 100, "source_record": "GPT 5.2 (xHigh reasoning)|" }, { "id": "aa947f3177959f4a4cd3", "model": "Kimi K3 (Max reasoning)", "organization": "Kimi", "configuration": "", "score": 0.376, "raw_score": 37.6, "raw_scale": 100, "source_record": "Kimi K3 (Max reasoning)|" }, { "id": "70b6ca766b6976e9e793", "model": "Gemini 3.5 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 0.368, "raw_score": 36.8, "raw_scale": 100, "source_record": "Gemini 3.5 Flash (High reasoning)|" }, { "id": "cad12193cdc170f04235", "model": "Gemini 3.1 Pro (High reasoning)", "organization": "Google", "configuration": "", "score": 0.336, "raw_score": 33.6, "raw_scale": 100, "source_record": "Gemini 3.1 Pro (High reasoning)|" }, { "id": "9331f45e6a98acfab5d0", "model": "Claude Opus 4.7 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.32799999999999996, "raw_score": 32.8, "raw_scale": 100, "source_record": "Claude Opus 4.7 (Adaptive/Max)|" }, { "id": "e73f710acdde2705c72f", "model": "Gemini 3.6 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 0.304, "raw_score": 30.4, "raw_scale": 100, "source_record": "Gemini 3.6 Flash (High reasoning)|" }, { "id": "e4c1fc09998792ebccd1", "model": "Muse Spark 1.3 (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 0.28, "raw_score": 28.0, "raw_scale": 100, "source_record": "Muse Spark 1.3 (xHigh reasoning)|" }, { "id": "7d6dfe8ab3bed6e659c4", "model": "Claude Opus 4.6 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.272, "raw_score": 27.2, "raw_scale": 100, "source_record": "Claude Opus 4.6 (Adaptive/Max)|" }, { "id": "63a68b209762d16ffd03", "model": "DeepSeek V4.1 Flash (Max reasoning)", "organization": "DeepSeek", "configuration": "", "score": 0.24, "raw_score": 24.0, "raw_scale": 100, "source_record": "DeepSeek V4.1 Flash (Max reasoning)|" }, { "id": "b447fc76574054444428", "model": "Hy4 Preview (High reasoning)", "organization": "Hunyuan", "configuration": "", "score": 0.24, "raw_score": 24.0, "raw_scale": 100, "source_record": "Hy4 Preview (High reasoning)|" }, { "id": "438fb619519085c3e804", "model": "DeepSeek V4 Flash Vision (experimental) (High reasoning)", "organization": "DeepSeek", "configuration": "", "score": 0.23199999999999998, "raw_score": 23.2, "raw_scale": 100, "source_record": "DeepSeek V4 Flash Vision (experimental) (High reasoning)|" }, { "id": "26db3aad46c4e1f70d40", "model": "Muse Spark 1.2 (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 0.23199999999999998, "raw_score": 23.2, "raw_scale": 100, "source_record": "Muse Spark 1.2 (xHigh reasoning)|" }, { "id": "bf1faa1f5b89c1ce19bc", "model": "Muse Spark 1.1 (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 0.20800000000000002, "raw_score": 20.8, "raw_scale": 100, "source_record": "Muse Spark 1.1 (xHigh reasoning)|" }, { "id": "f3a0f12d04e9997e8cdd", "model": "Inkling (xHigh reasoning)", "organization": "", "configuration": "", "score": 0.152, "raw_score": 15.2, "raw_scale": 100, "source_record": "Inkling (xHigh reasoning)|" }, { "id": "3ab24414f57d9cd8e66d", "model": "Qwen 3.7 Max (Thinking on)", "organization": "Qwen", "configuration": "", "score": 0.152, "raw_score": 15.2, "raw_scale": 100, "source_record": "Qwen 3.7 Max (Thinking on)|" }, { "id": "c920a6070ed97a981e45", "model": "Qwen 3.8 Max (xHigh reasoning)", "organization": "Qwen", "configuration": "", "score": 0.152, "raw_score": 15.2, "raw_scale": 100, "source_record": "Qwen 3.8 Max (xHigh reasoning)|" }, { "id": "e21c7a81768b80bca784", "model": "Muse Glimmer 30B (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 0.136, "raw_score": 13.6, "raw_scale": 100, "source_record": "Muse Glimmer 30B (xHigh reasoning)|" }, { "id": "9c79ade9bb3c4abd4997", "model": "GLM 5.3 (Max reasoning)", "organization": "Z.ai", "configuration": "", "score": 0.12, "raw_score": 12.0, "raw_scale": 100, "source_record": "GLM 5.3 (Max reasoning)|" }, { "id": "10c55109236838e3660d", "model": "Kimi K2.5 (Thinking on)", "organization": "Kimi", "configuration": "", "score": 0.12, "raw_score": 12.0, "raw_scale": 100, "source_record": "Kimi K2.5 (Thinking on)|" }, { "id": "a99246faae08a68de0ca", "model": "Claude Opus 4.5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.11199999999999999, "raw_score": 11.2, "raw_scale": 100, "source_record": "Claude Opus 4.5 (Adaptive/Max)|" }, { "id": "ed03b8c58a194e72d753", "model": "Gemini 3.5 Flash-Lite (High reasoning)", "organization": "Google", "configuration": "", "score": 0.11199999999999999, "raw_score": 11.2, "raw_scale": 100, "source_record": "Gemini 3.5 Flash-Lite (High reasoning)|" }, { "id": "cb353c8b680cb8c46e66", "model": "DeepSeek V4 Flash (Max reasoning)", "organization": "DeepSeek", "configuration": "", "score": 0.10400000000000001, "raw_score": 10.4, "raw_scale": 100, "source_record": "DeepSeek V4 Flash (Max reasoning)|" }, { "id": "d1f08003431327a00d9e", "model": "GLM 5.2 (Max reasoning)", "organization": "Z.ai", "configuration": "", "score": 0.10400000000000001, "raw_score": 10.4, "raw_scale": 100, "source_record": "GLM 5.2 (Max reasoning)|" }, { "id": "189c68ff156a5550a560", "model": "GLM 5.3 Flash (Max reasoning)", "organization": "Z.ai", "configuration": "", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "GLM 5.3 Flash (Max reasoning)|" }, { "id": "cf07097482234557cdef", "model": "DeepSeek V3.2 (High reasoning)", "organization": "DeepSeek", "configuration": "", "score": 0.08, "raw_score": 8.0, "raw_scale": 100, "source_record": "DeepSeek V3.2 (High reasoning)|" }, { "id": "54f8b24b586174a7ea02", "model": "Kimi K2.6 (Thinking on)", "organization": "Kimi", "configuration": "", "score": 0.07200000000000001, "raw_score": 7.2, "raw_scale": 100, "source_record": "Kimi K2.6 (Thinking on)|" }, { "id": "36ecd747db4944682896", "model": "DeepSeek V4 Pro (preview) (xHigh reasoning)", "organization": "DeepSeek", "configuration": "", "score": 0.055999999999999994, "raw_score": 5.6, "raw_scale": 100, "source_record": "DeepSeek V4 Pro (preview) (xHigh reasoning)|" }, { "id": "50441f579298f570c9b5", "model": "MAI Thinking 1", "organization": "Microsoft", "configuration": "", "score": 0.048, "raw_score": 4.8, "raw_scale": 100, "source_record": "MAI Thinking 1|" } ] }, { "id": "surge-chartography", "name": "Chartography", "group": "视觉与视频", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Professional graphical reasoning, official overall accuracy; model reasoning settings retained.", "zh": "专业图表推理，官方总体正确率；保留模型推理设置。" }, "source": { "name": "Surge AI", "benchmark_creator": "Surge AI", "url": "https://surgehq.ai/benchmarks/chartography", "data_url": "https://surgehq.ai/benchmarks/chartography", "sha256": "2200775d80273c6062f19768a2617d827a9b8e72cf957f870070d4cc084013d4", "retrieved_at": "2026-11-02T03:35:08.149647+00:00", "selector": "[data-leaderboard-item]: model label + numeric [data-score]", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "113ff3b4ddcb992525e3", "model": "Gemini 4 Argon (High reasoning)", "organization": "Google", "configuration": "", "score": 0.716, "raw_score": 71.6, "raw_scale": 100, "source_record": "Gemini 4 Argon (High reasoning)|" }, { "id": "f6cc94ca3090a4a25f2a", "model": "GPT 6 Astra (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.71, "raw_score": 71.0, "raw_scale": 100, "source_record": "GPT 6 Astra (Max reasoning)|" }, { "id": "58e21596a78517c854ca", "model": "Claude Opus 5.5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.6629999999999999, "raw_score": 66.3, "raw_scale": 100, "source_record": "Claude Opus 5.5 (Adaptive/Max)|" }, { "id": "e058db180145cdcf48e3", "model": "GPT 6 Sol (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.536, "raw_score": 53.6, "raw_scale": 100, "source_record": "GPT 6 Sol (Max reasoning)|" }, { "id": "1830705f32b90c041a16", "model": "Claude Fable 5.1 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.462, "raw_score": 46.2, "raw_scale": 100, "source_record": "Claude Fable 5.1 (Adaptive/Max)|" }, { "id": "9353e461a4affb00169c", "model": "GPT 5.6 Sol (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.45, "raw_score": 45.0, "raw_scale": 100, "source_record": "GPT 5.6 Sol (Max reasoning)|" }, { "id": "0f9f52222aa90b209a59", "model": "Gemini 3.8 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 0.409, "raw_score": 40.9, "raw_scale": 100, "source_record": "Gemini 3.8 Flash (High reasoning)|" }, { "id": "2b494471814b49a7c94b", "model": "Gemini 3.7 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 0.40399999999999997, "raw_score": 40.4, "raw_scale": 100, "source_record": "Gemini 3.7 Flash (High reasoning)|" }, { "id": "3fdad0139a57069b2d27", "model": "Claude Fable 5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.348, "raw_score": 34.8, "raw_scale": 100, "source_record": "Claude Fable 5 (Adaptive/Max)|" }, { "id": "f946a767b9d4ee5a5d63", "model": "GPT 5.6 Terra (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.34, "raw_score": 34.0, "raw_scale": 100, "source_record": "GPT 5.6 Terra (Max reasoning)|" }, { "id": "11af549aac5663809194", "model": "Gemini 3.6 Flash (Medium reasoning)", "organization": "Google", "configuration": "", "score": 0.34, "raw_score": 34.0, "raw_scale": 100, "source_record": "Gemini 3.6 Flash (Medium reasoning)|" }, { "id": "26db3aad46c4e1f70d40", "model": "Muse Spark 1.2 (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 0.321, "raw_score": 32.1, "raw_scale": 100, "source_record": "Muse Spark 1.2 (xHigh reasoning)|" }, { "id": "92ddc0fc7e504d35c9be", "model": "GPT 5.6 Luna (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.314, "raw_score": 31.4, "raw_scale": 100, "source_record": "GPT 5.6 Luna (Max reasoning)|" }, { "id": "6349b7a7be0d58473df5", "model": "GPT 5.5 (xHigh reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.31, "raw_score": 31.0, "raw_scale": 100, "source_record": "GPT 5.5 (xHigh reasoning)|" }, { "id": "ef7067e293c9801323de", "model": "GPT 5.4 (xHigh reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.29600000000000004, "raw_score": 29.6, "raw_scale": 100, "source_record": "GPT 5.4 (xHigh reasoning)|" }, { "id": "11202fdc5a0079debdd4", "model": "GPT 6 Luna (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "GPT 6 Luna (Max reasoning)|" }, { "id": "c920a6070ed97a981e45", "model": "Qwen 3.8 Max (xHigh reasoning)", "organization": "Qwen", "configuration": "", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "Qwen 3.8 Max (xHigh reasoning)|" }, { "id": "e4c1fc09998792ebccd1", "model": "Muse Spark 1.3 (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 0.276, "raw_score": 27.6, "raw_scale": 100, "source_record": "Muse Spark 1.3 (xHigh reasoning)|" }, { "id": "9db3a02f01736ec8d028", "model": "Claude Opus 5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.273, "raw_score": 27.3, "raw_scale": 100, "source_record": "Claude Opus 5 (Adaptive/Max)|" }, { "id": "aa947f3177959f4a4cd3", "model": "Kimi K3 (Max reasoning)", "organization": "Kimi", "configuration": "", "score": 0.266, "raw_score": 26.6, "raw_scale": 100, "source_record": "Kimi K3 (Max reasoning)|" }, { "id": "cad12193cdc170f04235", "model": "Gemini 3.1 Pro (High reasoning)", "organization": "Google", "configuration": "", "score": 0.261, "raw_score": 26.1, "raw_scale": 100, "source_record": "Gemini 3.1 Pro (High reasoning)|" }, { "id": "bf1faa1f5b89c1ce19bc", "model": "Muse Spark 1.1 (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 0.244, "raw_score": 24.4, "raw_scale": 100, "source_record": "Muse Spark 1.1 (xHigh reasoning)|" }, { "id": "db347ba49dfe3728a878", "model": "Grok 4.6 (xHigh reasoning)", "organization": "Grok", "configuration": "", "score": 0.205, "raw_score": 20.5, "raw_scale": 100, "source_record": "Grok 4.6 (xHigh reasoning)|" }, { "id": "c6802335cd539486df79", "model": "Qwen 3.8 Flash (xHigh reasoning)", "organization": "Qwen", "configuration": "", "score": 0.174, "raw_score": 17.4, "raw_scale": 100, "source_record": "Qwen 3.8 Flash (xHigh reasoning)|" }, { "id": "e21c7a81768b80bca784", "model": "Muse Glimmer 30B (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 0.172, "raw_score": 17.2, "raw_scale": 100, "source_record": "Muse Glimmer 30B (xHigh reasoning)|" }, { "id": "81896f03145bc246072e", "model": "Grok 4.5 (High reasoning)", "organization": "Grok", "configuration": "", "score": 0.17, "raw_score": 17.0, "raw_scale": 100, "source_record": "Grok 4.5 (High reasoning)|" }, { "id": "001141cb430b0c7b612e", "model": "Claude Sonnet 5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.166, "raw_score": 16.6, "raw_scale": 100, "source_record": "Claude Sonnet 5 (Adaptive/Max)|" }, { "id": "9331f45e6a98acfab5d0", "model": "Claude Opus 4.7 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.165, "raw_score": 16.5, "raw_scale": 100, "source_record": "Claude Opus 4.7 (Adaptive/Max)|" }, { "id": "189c68ff156a5550a560", "model": "GLM 5.3 Flash (Max reasoning)", "organization": "Z.ai", "configuration": "", "score": 0.163, "raw_score": 16.3, "raw_scale": 100, "source_record": "GLM 5.3 Flash (Max reasoning)|" }, { "id": "8f8ce8676556b80fa20e", "model": "Claude Opus 4.8 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 0.159, "raw_score": 15.9, "raw_scale": 100, "source_record": "Claude Opus 4.8 (Adaptive/Max)|" }, { "id": "1e4119a49b30d4260eac", "model": "Qwen 3.5 Plus (Thinking on)", "organization": "Qwen", "configuration": "", "score": 0.159, "raw_score": 15.9, "raw_scale": 100, "source_record": "Qwen 3.5 Plus (Thinking on)|" }, { "id": "3c04f1378a544ed3497c", "model": "Qwen 3.7 Plus (Thinking on)", "organization": "Qwen", "configuration": "", "score": 0.158, "raw_score": 15.8, "raw_scale": 100, "source_record": "Qwen 3.7 Plus (Thinking on)|" }, { "id": "da2bab3b3779a0f08ab2", "model": "Grok 4.7 (xHigh reasoning)", "organization": "Grok", "configuration": "", "score": 0.147, "raw_score": 14.7, "raw_scale": 100, "source_record": "Grok 4.7 (xHigh reasoning)|" }, { "id": "9b0f2966ebfeff7d46b8", "model": "Grok 4.3 (High reasoning)", "organization": "Grok", "configuration": "", "score": 0.129, "raw_score": 12.9, "raw_scale": 100, "source_record": "Grok 4.3 (High reasoning)|" }, { "id": "10c55109236838e3660d", "model": "Kimi K2.5 (Thinking on)", "organization": "Kimi", "configuration": "", "score": 0.126, "raw_score": 12.6, "raw_scale": 100, "source_record": "Kimi K2.5 (Thinking on)|" }, { "id": "54f8b24b586174a7ea02", "model": "Kimi K2.6 (Thinking on)", "organization": "Kimi", "configuration": "", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "Kimi K2.6 (Thinking on)|" }, { "id": "63a68b209762d16ffd03", "model": "DeepSeek V4.1 Flash (Max reasoning)", "organization": "DeepSeek", "configuration": "", "score": 0.11599999999999999, "raw_score": 11.6, "raw_scale": 100, "source_record": "DeepSeek V4.1 Flash (Max reasoning)|" }, { "id": "f0bbea4bc0a916fb34fd", "model": "DeepSeek V4 Flash Vision (experimental) (Max reasoning)", "organization": "DeepSeek", "configuration": "", "score": 0.11199999999999999, "raw_score": 11.2, "raw_scale": 100, "source_record": "DeepSeek V4 Flash Vision (experimental) (Max reasoning)|" }, { "id": "7e0b9b061f574fb2c18e", "model": "Gemini 3.5 Flash-Lite (Minimal reasoning)", "organization": "Google", "configuration": "", "score": 0.102, "raw_score": 10.2, "raw_scale": 100, "source_record": "Gemini 3.5 Flash-Lite (Minimal reasoning)|" }, { "id": "f0b455e5e9660af9ff40", "model": "Inkling (High reasoning)", "organization": "", "configuration": "", "score": 0.1, "raw_score": 10.0, "raw_scale": 100, "source_record": "Inkling (High reasoning)|" }, { "id": "ba9748f2bf0eb8ad0e45", "model": "Inkling Small (xHigh reasoning)", "organization": "", "configuration": "", "score": 0.091, "raw_score": 9.1, "raw_scale": 100, "source_record": "Inkling Small (xHigh reasoning)|" }, { "id": "408b5ebcbcaac5754487", "model": "Mistral Large 3", "organization": "Mistral", "configuration": "", "score": 0.09, "raw_score": 9.0, "raw_scale": 100, "source_record": "Mistral Large 3|" } ] }, { "id": "surge-enterprise", "name": "EnterpriseBench: CoreCraft", "group": "专业工作", "unit": "score", "metric": "Mean task score", "protocol": { "en": "Official CoreCraft overall task score on the original 0–100 scale; partial credit is not a task pass rate.", "zh": "CoreCraft 官方总体任务得分，保留 0–100 原量纲；部分得分不等于任务通过率。" }, "source": { "name": "Surge AI", "benchmark_creator": "Surge AI", "url": "https://surgehq.ai/benchmarks/enterprisebench-corecraft", "data_url": "https://surgehq.ai/benchmarks/enterprisebench-corecraft", "sha256": "f437f98167d65ba2bd8220124e2071499d12f70131981e9a836c3acfe3e2bd86", "retrieved_at": "2026-11-02T03:35:08.245179+00:00", "selector": "[data-leaderboard-item]: model label + numeric [data-score]", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "1830705f32b90c041a16", "model": "Claude Fable 5.1 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 77.4, "raw_score": 77.4, "raw_scale": 1, "source_record": "Claude Fable 5.1 (Adaptive/Max)|" }, { "id": "58e21596a78517c854ca", "model": "Claude Opus 5.5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 74.4, "raw_score": 74.4, "raw_scale": 1, "source_record": "Claude Opus 5.5 (Adaptive/Max)|" }, { "id": "3fdad0139a57069b2d27", "model": "Claude Fable 5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 70.3, "raw_score": 70.3, "raw_scale": 1, "source_record": "Claude Fable 5 (Adaptive/Max)|" }, { "id": "9db3a02f01736ec8d028", "model": "Claude Opus 5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 68.7, "raw_score": 68.7, "raw_scale": 1, "source_record": "Claude Opus 5 (Adaptive/Max)|" }, { "id": "da2bab3b3779a0f08ab2", "model": "Grok 4.7 (xHigh reasoning)", "organization": "Grok", "configuration": "", "score": 68.2, "raw_score": 68.2, "raw_scale": 1, "source_record": "Grok 4.7 (xHigh reasoning)|" }, { "id": "db347ba49dfe3728a878", "model": "Grok 4.6 (xHigh reasoning)", "organization": "Grok", "configuration": "", "score": 65.6, "raw_score": 65.6, "raw_scale": 1, "source_record": "Grok 4.6 (xHigh reasoning)|" }, { "id": "b447fc76574054444428", "model": "Hy4 Preview (High reasoning)", "organization": "Hunyuan", "configuration": "", "score": 64.6, "raw_score": 64.6, "raw_scale": 1, "source_record": "Hy4 Preview (High reasoning)|" }, { "id": "e4c1fc09998792ebccd1", "model": "Muse Spark 1.3 (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 61.9, "raw_score": 61.9, "raw_scale": 1, "source_record": "Muse Spark 1.3 (xHigh reasoning)|" }, { "id": "0f9f52222aa90b209a59", "model": "Gemini 3.8 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 58.5, "raw_score": 58.5, "raw_scale": 1, "source_record": "Gemini 3.8 Flash (High reasoning)|" }, { "id": "c6802335cd539486df79", "model": "Qwen 3.8 Flash (xHigh reasoning)", "organization": "Qwen", "configuration": "", "score": 57.4, "raw_score": 57.4, "raw_scale": 1, "source_record": "Qwen 3.8 Flash (xHigh reasoning)|" }, { "id": "2b494471814b49a7c94b", "model": "Gemini 3.7 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 55.9, "raw_score": 55.9, "raw_scale": 1, "source_record": "Gemini 3.7 Flash (High reasoning)|" }, { "id": "d22e89e76852891376da", "model": "DeepSeek V4 Pro (Max reasoning)", "organization": "DeepSeek", "configuration": "", "score": 53.3, "raw_score": 53.3, "raw_scale": 1, "source_record": "DeepSeek V4 Pro (Max reasoning)|" }, { "id": "8f8ce8676556b80fa20e", "model": "Claude Opus 4.8 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 52.3, "raw_score": 52.3, "raw_scale": 1, "source_record": "Claude Opus 4.8 (Adaptive/Max)|" }, { "id": "9c79ade9bb3c4abd4997", "model": "GLM 5.3 (Max reasoning)", "organization": "Z.ai", "configuration": "", "score": 51.8, "raw_score": 51.8, "raw_scale": 1, "source_record": "GLM 5.3 (Max reasoning)|" }, { "id": "e21c7a81768b80bca784", "model": "Muse Glimmer 30B (xHigh reasoning)", "organization": "Meta", "configuration": "", "score": 51.8, "raw_score": 51.8, "raw_scale": 1, "source_record": "Muse Glimmer 30B (xHigh reasoning)|" }, { "id": "001141cb430b0c7b612e", "model": "Claude Sonnet 5 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 51.3, "raw_score": 51.3, "raw_scale": 1, "source_record": "Claude Sonnet 5 (Adaptive/Max)|" }, { "id": "6349b7a7be0d58473df5", "model": "GPT 5.5 (xHigh reasoning)", "organization": "OpenAI", "configuration": "", "score": 51.3, "raw_score": 51.3, "raw_scale": 1, "source_record": "GPT 5.5 (xHigh reasoning)|" }, { "id": "70b6ca766b6976e9e793", "model": "Gemini 3.5 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 50.8, "raw_score": 50.8, "raw_scale": 1, "source_record": "Gemini 3.5 Flash (High reasoning)|" }, { "id": "189c68ff156a5550a560", "model": "GLM 5.3 Flash (Max reasoning)", "organization": "Z.ai", "configuration": "", "score": 46.7, "raw_score": 46.7, "raw_scale": 1, "source_record": "GLM 5.3 Flash (Max reasoning)|" }, { "id": "9353e461a4affb00169c", "model": "GPT 5.6 Sol (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 46.2, "raw_score": 46.2, "raw_scale": 1, "source_record": "GPT 5.6 Sol (Max reasoning)|" }, { "id": "f0bbea4bc0a916fb34fd", "model": "DeepSeek V4 Flash Vision (experimental) (Max reasoning)", "organization": "DeepSeek", "configuration": "", "score": 45.6, "raw_score": 45.6, "raw_scale": 1, "source_record": "DeepSeek V4 Flash Vision (experimental) (Max reasoning)|" }, { "id": "11202fdc5a0079debdd4", "model": "GPT 6 Luna (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 43.1, "raw_score": 43.1, "raw_scale": 1, "source_record": "GPT 6 Luna (Max reasoning)|" }, { "id": "e058db180145cdcf48e3", "model": "GPT 6 Sol (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 43.1, "raw_score": 43.1, "raw_scale": 1, "source_record": "GPT 6 Sol (Max reasoning)|" }, { "id": "aa947f3177959f4a4cd3", "model": "Kimi K3 (Max reasoning)", "organization": "Kimi", "configuration": "", "score": 43.1, "raw_score": 43.1, "raw_scale": 1, "source_record": "Kimi K3 (Max reasoning)|" }, { "id": "188c7f4fc3e592a8f02c", "model": "GPT 5.2 (xHigh reasoning)", "organization": "OpenAI", "configuration": "", "score": 42.6, "raw_score": 42.6, "raw_scale": 1, "source_record": "GPT 5.2 (xHigh reasoning)|" }, { "id": "f6cc94ca3090a4a25f2a", "model": "GPT 6 Astra (Max reasoning)", "organization": "OpenAI", "configuration": "", "score": 42.6, "raw_score": 42.6, "raw_scale": 1, "source_record": "GPT 6 Astra (Max reasoning)|" }, { "id": "f41502a212d1aaebf379", "model": "GLM 5.2 (xHigh reasoning)", "organization": "Z.ai", "configuration": "", "score": 42.1, "raw_score": 42.1, "raw_scale": 1, "source_record": "GLM 5.2 (xHigh reasoning)|" }, { "id": "63a68b209762d16ffd03", "model": "DeepSeek V4.1 Flash (Max reasoning)", "organization": "DeepSeek", "configuration": "", "score": 41.5, "raw_score": 41.5, "raw_scale": 1, "source_record": "DeepSeek V4.1 Flash (Max reasoning)|" }, { "id": "5bd03a0a9a48adcebc86", "model": "Hy3 (High reasoning)", "organization": "Hunyuan", "configuration": "", "score": 41.0, "raw_score": 41.0, "raw_scale": 1, "source_record": "Hy3 (High reasoning)|" }, { "id": "e73f710acdde2705c72f", "model": "Gemini 3.6 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 39.5, "raw_score": 39.5, "raw_scale": 1, "source_record": "Gemini 3.6 Flash (High reasoning)|" }, { "id": "ef7067e293c9801323de", "model": "GPT 5.4 (xHigh reasoning)", "organization": "OpenAI", "configuration": "", "score": 36.4, "raw_score": 36.4, "raw_scale": 1, "source_record": "GPT 5.4 (xHigh reasoning)|" }, { "id": "9331f45e6a98acfab5d0", "model": "Claude Opus 4.7 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 35.9, "raw_score": 35.9, "raw_scale": 1, "source_record": "Claude Opus 4.7 (Adaptive/Max)|" }, { "id": "7d6dfe8ab3bed6e659c4", "model": "Claude Opus 4.6 (Adaptive/Max)", "organization": "Anthropic", "configuration": "", "score": 30.8, "raw_score": 30.8, "raw_scale": 1, "source_record": "Claude Opus 4.6 (Adaptive/Max)|" }, { "id": "084090120e9848a69d3e", "model": "DeepSeek V4 Flash (High reasoning)", "organization": "DeepSeek", "configuration": "", "score": 30.8, "raw_score": 30.8, "raw_scale": 1, "source_record": "DeepSeek V4 Flash (High reasoning)|" }, { "id": "cad12193cdc170f04235", "model": "Gemini 3.1 Pro (High reasoning)", "organization": "Google", "configuration": "", "score": 27.2, "raw_score": 27.2, "raw_scale": 1, "source_record": "Gemini 3.1 Pro (High reasoning)|" }, { "id": "3ab24414f57d9cd8e66d", "model": "Qwen 3.7 Max (Thinking on)", "organization": "Qwen", "configuration": "", "score": 26.2, "raw_score": 26.2, "raw_scale": 1, "source_record": "Qwen 3.7 Max (Thinking on)|" }, { "id": "54f8b24b586174a7ea02", "model": "Kimi K2.6 (Thinking on)", "organization": "Kimi", "configuration": "", "score": 24.6, "raw_score": 24.6, "raw_scale": 1, "source_record": "Kimi K2.6 (Thinking on)|" }, { "id": "cf07097482234557cdef", "model": "DeepSeek V3.2 (High reasoning)", "organization": "DeepSeek", "configuration": "", "score": 24.1, "raw_score": 24.1, "raw_scale": 1, "source_record": "DeepSeek V3.2 (High reasoning)|" }, { "id": "bedc04dd93c523649dfc", "model": "Grok 4.1 (Fast)", "organization": "Grok", "configuration": "", "score": 20.5, "raw_score": 20.5, "raw_scale": 1, "source_record": "Grok 4.1 (Fast)|" }, { "id": "ec871080337a023f1127", "model": "GPT 5.2 Codex (xHigh reasoning)", "organization": "OpenAI", "configuration": "", "score": 20.1, "raw_score": 20.1, "raw_scale": 1, "source_record": "GPT 5.2 Codex (xHigh reasoning)|" }, { "id": "ba646870fe9bb67c04ae", "model": "Gemini 3 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 20.0, "raw_score": 20.0, "raw_scale": 1, "source_record": "Gemini 3 Flash (High reasoning)|" }, { "id": "258094eb25af7a0653a7", "model": "GLM 5 (Auto reasoning)", "organization": "Z.ai", "configuration": "", "score": 17.4, "raw_score": 17.4, "raw_scale": 1, "source_record": "GLM 5 (Auto reasoning)|" }, { "id": "e95ffc8e5e82c8058f39", "model": "Claude Sonnet 4.6 (High reasoning)", "organization": "Anthropic", "configuration": "", "score": 16.4, "raw_score": 16.4, "raw_scale": 1, "source_record": "Claude Sonnet 4.6 (High reasoning)|" }, { "id": "6b63f34f5c77ab89f420", "model": "Gemini 3 Pro (High reasoning)", "organization": "Google", "configuration": "", "score": 14.4, "raw_score": 14.4, "raw_scale": 1, "source_record": "Gemini 3 Pro (High reasoning)|" }, { "id": "1e4119a49b30d4260eac", "model": "Qwen 3.5 Plus (Thinking on)", "organization": "Qwen", "configuration": "", "score": 11.3, "raw_score": 11.3, "raw_scale": 1, "source_record": "Qwen 3.5 Plus (Thinking on)|" }, { "id": "63c7f16712c47f9b6bd5", "model": "Nemotron Lightning 3.5 30B A3B (Thinking on)", "organization": "NVIDIA", "configuration": "", "score": 10.3, "raw_score": 10.3, "raw_scale": 1, "source_record": "Nemotron Lightning 3.5 30B A3B (Thinking on)|" }, { "id": "1ee5d66ea6e4632a0ec6", "model": "Nova 2 Pro (High reasoning)", "organization": "Amazon", "configuration": "", "score": 8.9, "raw_score": 8.9, "raw_scale": 1, "source_record": "Nova 2 Pro (High reasoning)|" }, { "id": "10c55109236838e3660d", "model": "Kimi K2.5 (Thinking on)", "organization": "Kimi", "configuration": "", "score": 8.7, "raw_score": 8.7, "raw_scale": 1, "source_record": "Kimi K2.5 (Thinking on)|" }, { "id": "408b5ebcbcaac5754487", "model": "Mistral Large 3", "organization": "Mistral", "configuration": "", "score": 3.6, "raw_score": 3.6, "raw_scale": 1, "source_record": "Mistral Large 3|" }, { "id": "864dcec3e9670b55ae15", "model": "Qwen 3 Max (Thinking on)", "organization": "Qwen", "configuration": "", "score": 3.6, "raw_score": 3.6, "raw_scale": 1, "source_record": "Qwen 3 Max (Thinking on)|" } ], "score_max": 100 }, { "id": "surge-hemingway", "name": "Hemingway-bench", "group": "生成与创作", "unit": "elo", "metric": "Elo", "protocol": { "en": "Published writing-preference Elo; not a percentage or task completion rate.", "zh": "官方写作偏好 Elo；不属于百分制或任务通过率。" }, "source": { "name": "Surge AI", "benchmark_creator": "Surge AI", "url": "https://surgehq.ai/benchmarks/hemingway-bench", "data_url": "https://surgehq.ai/benchmarks/hemingway-bench", "sha256": "cf70b97648c5412196abd0e9bda8c988d562eab4f73bf19beab926c3ca766ca1", "retrieved_at": "2026-11-02T03:35:08.285121+00:00", "selector": "[data-leaderboard-item]: model label + numeric [data-score]", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "1ca5988bbc6fe176b5a3", "model": "Claude Fable 5 (Adaptive/High)", "organization": "Anthropic", "configuration": "", "score": 1110.0, "raw_score": 1110.0, "raw_scale": 1, "source_record": "Claude Fable 5 (Adaptive/High)|" }, { "id": "13801fbcec61c6aa0e44", "model": "Gemini 3.7 Flash (Medium reasoning)", "organization": "Google", "configuration": "", "score": 1105.0, "raw_score": 1105.0, "raw_scale": 1, "source_record": "Gemini 3.7 Flash (Medium reasoning)|" }, { "id": "bc09a688f372c2ddd583", "model": "Claude Fable 5.1 (Adaptive/High)", "organization": "Anthropic", "configuration": "", "score": 1095.0, "raw_score": 1095.0, "raw_scale": 1, "source_record": "Claude Fable 5.1 (Adaptive/High)|" }, { "id": "e38b3e4a7d390564e217", "model": "Gemini 3.8 Flash (Medium reasoning)", "organization": "Google", "configuration": "", "score": 1093.0, "raw_score": 1093.0, "raw_scale": 1, "source_record": "Gemini 3.8 Flash (Medium reasoning)|" }, { "id": "11af549aac5663809194", "model": "Gemini 3.6 Flash (Medium reasoning)", "organization": "Google", "configuration": "", "score": 1075.0, "raw_score": 1075.0, "raw_scale": 1, "source_record": "Gemini 3.6 Flash (Medium reasoning)|" }, { "id": "cad12193cdc170f04235", "model": "Gemini 3.1 Pro (High reasoning)", "organization": "Google", "configuration": "", "score": 1073.0, "raw_score": 1073.0, "raw_scale": 1, "source_record": "Gemini 3.1 Pro (High reasoning)|" }, { "id": "916a68c145b9fe0d8ee4", "model": "Gemini 3.5 Flash (Medium reasoning)", "organization": "Google", "configuration": "", "score": 1071.0, "raw_score": 1071.0, "raw_scale": 1, "source_record": "Gemini 3.5 Flash (Medium reasoning)|" }, { "id": "aa947f3177959f4a4cd3", "model": "Kimi K3 (Max reasoning)", "organization": "Kimi", "configuration": "", "score": 1069.0, "raw_score": 1069.0, "raw_scale": 1, "source_record": "Kimi K3 (Max reasoning)|" }, { "id": "aef5c90d8e2bdf5d7cc9", "model": "GPT 5.6 Sol (Medium reasoning)", "organization": "OpenAI", "configuration": "", "score": 1057.0, "raw_score": 1057.0, "raw_scale": 1, "source_record": "GPT 5.6 Sol (Medium reasoning)|" }, { "id": "6b63f34f5c77ab89f420", "model": "Gemini 3 Pro (High reasoning)", "organization": "Google", "configuration": "", "score": 1056.0, "raw_score": 1056.0, "raw_scale": 1, "source_record": "Gemini 3 Pro (High reasoning)|" }, { "id": "9c79ade9bb3c4abd4997", "model": "GLM 5.3 (Max reasoning)", "organization": "Z.ai", "configuration": "", "score": 1053.0, "raw_score": 1053.0, "raw_scale": 1, "source_record": "GLM 5.3 (Max reasoning)|" }, { "id": "ba646870fe9bb67c04ae", "model": "Gemini 3 Flash (High reasoning)", "organization": "Google", "configuration": "", "score": 1052.0, "raw_score": 1052.0, "raw_scale": 1, "source_record": "Gemini 3 Flash (High reasoning)|" }, { "id": "cea8498438dd16f0c88b", "model": "Claude Opus 5 (Adaptive/High)", "organization": "Anthropic", "configuration": "", "score": 1047.0, "raw_score": 1047.0, "raw_scale": 1, "source_record": "Claude Opus 5 (Adaptive/High)|" }, { "id": "ca6117e2f4678331a7d6", "model": "DeepSeek V4 Pro (High reasoning)", "organization": "DeepSeek", "configuration": "", "score": 1046.0, "raw_score": 1046.0, "raw_scale": 1, "source_record": "DeepSeek V4 Pro (High reasoning)|" }, { "id": "441d90f8be1049dcc583", "model": "Claude Opus 4.6 (High reasoning)", "organization": "Anthropic", "configuration": "", "score": 1045.0, "raw_score": 1045.0, "raw_scale": 1, "source_record": "Claude Opus 4.6 (High reasoning)|" }, { "id": "9af1fb27fb70e4ddfee5", "model": "Claude Opus 4.8 (Adaptive/Default/High)", "organization": "Anthropic", "configuration": "", "score": 1044.0, "raw_score": 1044.0, "raw_scale": 1, "source_record": "Claude Opus 4.8 (Adaptive/Default/High)|" }, { "id": "4974458e0f37487f20f5", "model": "Claude Opus 4.7 (High reasoning)", "organization": "Anthropic", "configuration": "", "score": 1042.0, "raw_score": 1042.0, "raw_scale": 1, "source_record": "Claude Opus 4.7 (High reasoning)|" }, { "id": "125ef25f78810d27e2e3", "model": "Claude Opus 4.8 (High reasoning)", "organization": "Anthropic", "configuration": "", "score": 1038.0, "raw_score": 1038.0, "raw_scale": 1, "source_record": "Claude Opus 4.8 (High reasoning)|" }, { "id": "d5743d4d0ca1c86d7f3b", "model": "GLM 5.2 (Auto reasoning)", "organization": "Z.ai", "configuration": "", "score": 1036.0, "raw_score": 1036.0, "raw_scale": 1, "source_record": "GLM 5.2 (Auto reasoning)|" }, { "id": "a476195f79af38e851a1", "model": "GPT 5.5 (Medium reasoning)", "organization": "OpenAI", "configuration": "", "score": 1035.0, "raw_score": 1035.0, "raw_scale": 1, "source_record": "GPT 5.5 (Medium reasoning)|" }, { "id": "189c68ff156a5550a560", "model": "GLM 5.3 Flash (Max reasoning)", "organization": "Z.ai", "configuration": "", "score": 1034.0, "raw_score": 1034.0, "raw_scale": 1, "source_record": "GLM 5.3 Flash (Max reasoning)|" }, { "id": "54f8b24b586174a7ea02", "model": "Kimi K2.6 (Thinking on)", "organization": "Kimi", "configuration": "", "score": 1021.0, "raw_score": 1021.0, "raw_scale": 1, "source_record": "Kimi K2.6 (Thinking on)|" }, { "id": "5b685b16832ec787960d", "model": "Claude Opus 4.5 (High reasoning)", "organization": "Anthropic", "configuration": "", "score": 1016.0, "raw_score": 1016.0, "raw_scale": 1, "source_record": "Claude Opus 4.5 (High reasoning)|" }, { "id": "b447fc76574054444428", "model": "Hy4 Preview (High reasoning)", "organization": "Hunyuan", "configuration": "", "score": 1014.0, "raw_score": 1014.0, "raw_scale": 1, "source_record": "Hy4 Preview (High reasoning)|" }, { "id": "ef45507ddedf31ea589d", "model": "Muse Spark 1.2 (Auto reasoning)", "organization": "Meta", "configuration": "", "score": 1013.0, "raw_score": 1013.0, "raw_scale": 1, "source_record": "Muse Spark 1.2 (Auto reasoning)|" }, { "id": "c920a6070ed97a981e45", "model": "Qwen 3.8 Max (xHigh reasoning)", "organization": "Qwen", "configuration": "", "score": 1009.0, "raw_score": 1009.0, "raw_scale": 1, "source_record": "Qwen 3.8 Max (xHigh reasoning)|" }, { "id": "e95ffc8e5e82c8058f39", "model": "Claude Sonnet 4.6 (High reasoning)", "organization": "Anthropic", "configuration": "", "score": 1001.0, "raw_score": 1001.0, "raw_scale": 1, "source_record": "Claude Sonnet 4.6 (High reasoning)|" }, { "id": "df750963cb97c3eba18d", "model": "DeepSeek V4 Pro (preview) (High reasoning)", "organization": "DeepSeek", "configuration": "", "score": 999.0, "raw_score": 999.0, "raw_scale": 1, "source_record": "DeepSeek V4 Pro (preview) (High reasoning)|" }, { "id": "066bf4a3c8bbff767753", "model": "GPT 5.2 Chat", "organization": "OpenAI", "configuration": "", "score": 994.0, "raw_score": 994.0, "raw_scale": 1, "source_record": "GPT 5.2 Chat|" }, { "id": "7e0b9b061f574fb2c18e", "model": "Gemini 3.5 Flash-Lite (Minimal reasoning)", "organization": "Google", "configuration": "", "score": 990.0, "raw_score": 990.0, "raw_scale": 1, "source_record": "Gemini 3.5 Flash-Lite (Minimal reasoning)|" }, { "id": "1e4119a49b30d4260eac", "model": "Qwen 3.5 Plus (Thinking on)", "organization": "Qwen", "configuration": "", "score": 990.0, "raw_score": 990.0, "raw_scale": 1, "source_record": "Qwen 3.5 Plus (Thinking on)|" }, { "id": "10c55109236838e3660d", "model": "Kimi K2.5 (Thinking on)", "organization": "Kimi", "configuration": "", "score": 988.0, "raw_score": 988.0, "raw_scale": 1, "source_record": "Kimi K2.5 (Thinking on)|" }, { "id": "1481a639cb9eb1fafc6e", "model": "GPT 5.4 (No reasoning)", "organization": "OpenAI", "configuration": "", "score": 985.0, "raw_score": 985.0, "raw_scale": 1, "source_record": "GPT 5.4 (No reasoning)|" }, { "id": "1a9d454e610f0a67c01f", "model": "Muse Spark 1.1 (Medium reasoning)", "organization": "Meta", "configuration": "", "score": 985.0, "raw_score": 985.0, "raw_scale": 1, "source_record": "Muse Spark 1.1 (Medium reasoning)|" }, { "id": "c633699256d1ba0e6af4", "model": "Grok 4.6 (High reasoning)", "organization": "Grok", "configuration": "", "score": 983.0, "raw_score": 983.0, "raw_scale": 1, "source_record": "Grok 4.6 (High reasoning)|" }, { "id": "56156fb16e638bc84f2f", "model": "DeepSeek V4 Flash Vision (experimental)", "organization": "DeepSeek", "configuration": "", "score": 981.0, "raw_score": 981.0, "raw_scale": 1, "source_record": "DeepSeek V4 Flash Vision (experimental)|" }, { "id": "81896f03145bc246072e", "model": "Grok 4.5 (High reasoning)", "organization": "Grok", "configuration": "", "score": 972.0, "raw_score": 972.0, "raw_scale": 1, "source_record": "Grok 4.5 (High reasoning)|" }, { "id": "a2b438c36f5d80dd31a8", "model": "Hy3 (No reasoning)", "organization": "Hunyuan", "configuration": "", "score": 971.0, "raw_score": 971.0, "raw_scale": 1, "source_record": "Hy3 (No reasoning)|" }, { "id": "9939af0058dcddfc6c2c", "model": "DeepSeek V4 Flash (preview) (High reasoning)", "organization": "DeepSeek", "configuration": "", "score": 969.0, "raw_score": 969.0, "raw_scale": 1, "source_record": "DeepSeek V4 Flash (preview) (High reasoning)|" }, { "id": "766778c15d8fefb5f348", "model": "GPT 5.2 (No reasoning)", "organization": "OpenAI", "configuration": "", "score": 954.0, "raw_score": 954.0, "raw_scale": 1, "source_record": "GPT 5.2 (No reasoning)|" }, { "id": "9caabd90b73ebda6412d", "model": "Qwen 3 Max (No reasoning)", "organization": "Qwen", "configuration": "", "score": 948.0, "raw_score": 948.0, "raw_scale": 1, "source_record": "Qwen 3 Max (No reasoning)|" }, { "id": "c6802335cd539486df79", "model": "Qwen 3.8 Flash (xHigh reasoning)", "organization": "Qwen", "configuration": "", "score": 929.0, "raw_score": 929.0, "raw_scale": 1, "source_record": "Qwen 3.8 Flash (xHigh reasoning)|" }, { "id": "ec83fe06945f55e8f36b", "model": "Grok 4.1 Fast Reasoning (Thinking on)", "organization": "Grok", "configuration": "", "score": 916.0, "raw_score": 916.0, "raw_scale": 1, "source_record": "Grok 4.1 Fast Reasoning (Thinking on)|" }, { "id": "edc151e7abfe4d14629b", "model": "Muse Glimmer 30B (High reasoning)", "organization": "Meta", "configuration": "", "score": 913.0, "raw_score": 913.0, "raw_scale": 1, "source_record": "Muse Glimmer 30B (High reasoning)|" }, { "id": "63c7f16712c47f9b6bd5", "model": "Nemotron Lightning 3.5 30B A3B (Thinking on)", "organization": "NVIDIA", "configuration": "", "score": 888.0, "raw_score": 888.0, "raw_scale": 1, "source_record": "Nemotron Lightning 3.5 30B A3B (Thinking on)|" }, { "id": "aa5bcd89ecb98b39f0e8", "model": "Kimi K2 Instruct", "organization": "Kimi", "configuration": "", "score": 883.0, "raw_score": 883.0, "raw_scale": 1, "source_record": "Kimi K2 Instruct|" }, { "id": "7902be7c08658bd5bf22", "model": "Llama 4 Maverick", "organization": "Meta", "configuration": "", "score": 803.0, "raw_score": 803.0, "raw_scale": 1, "source_record": "Llama 4 Maverick|" }, { "id": "1c2537307e0fe8af1178", "model": "Nova 2 Pro (No reasoning)", "organization": "Amazon", "configuration": "", "score": 749.0, "raw_score": 749.0, "raw_scale": 1, "source_record": "Nova 2 Pro (No reasoning)|" } ] }, { "id": "apex-agents-mercor", "name": "APEX-Agents v1.1 (Mercor)", "group": "专业工作", "unit": "%", "metric": "pass-1", "protocol": { "en": "Mercor global leaderboard; complete task pass@1. Each model/reasoning/harness configuration remains separate.", "zh": "Mercor 总榜；完整任务 pass@1。各模型、推理与 Agent 配置分别记录。" }, "source": { "name": "Mercor", "benchmark_creator": "Mercor", "url": "https://www.mercor.com/apex/", "data_url": "https://www.mercor.com/apex/", "sha256": "9a1f8200f24bae14929d7d2cf9129f85b338b131811f0f93bfc86c8aff535edc", "retrieved_at": "2026-11-02T03:34:24.031268+00:00", "selector": "__NEXT_DATA__.props.pageProps.benchmarks[benchmarkId=apex-agents].globalLeaderboard; pass-1; harnessScores", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "2b62e1f0450fa87aa17e", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.8420000000000001, "raw_score": 84.2, "raw_scale": 100, "source_record": "gemini-4-carbon|loop_truncated_tools_agent|pass-1", "source_model_id": "gemini-4-carbon", "n_samples": 960, "reported_error": 4.3 }, { "id": "c205f1d6e4ec7787d84a", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.8240000000000001, "raw_score": 82.4, "raw_scale": 100, "source_record": "gemini-4-argon|loop_truncated_tools_agent|pass-1", "source_model_id": "gemini-4-argon", "n_samples": 954, "reported_error": 4.3 }, { "id": "ca3308b9b860f0456282", "model": "Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.755, "raw_score": 75.5, "raw_scale": 100, "source_record": "claude-sonnet-5-5|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-sonnet-5-5", "n_samples": 955, "reported_error": 4.7 }, { "id": "b777c6cd0d5458146543", "model": "Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.735, "raw_score": 73.5, "raw_scale": 100, "source_record": "claude-opus-5-5|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-opus-5-5", "n_samples": 950, "reported_error": 4.9 }, { "id": "37534f494cf8159a2f30", "model": "DeepSeek-V4.1-Flash-1017", "organization": "DeepSeek", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.698, "raw_score": 69.8, "raw_scale": 100, "source_record": "deepseek-v4-1-flash-1017|loop_truncated_tools_agent|pass-1", "source_model_id": "deepseek-v4-1-flash-1017", "n_samples": 957, "reported_error": 4.9 }, { "id": "49b4b7d782d5977bb0cf", "model": "Kimi K3.1 Pro", "organization": "Kimi", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.6940000000000001, "raw_score": 69.4, "raw_scale": 100, "source_record": "kimi-k3.1-pro|loop_truncated_tools_agent|pass-1", "source_model_id": "kimi-k3.1-pro", "n_samples": 952, "reported_error": 5 }, { "id": "d5baf5cf7043b6e3a11b", "model": "Fable 5.1", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.6859999999999999, "raw_score": 68.6, "raw_scale": 100, "source_record": "claude-fable-5.1|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-fable-5.1", "n_samples": 959, "reported_error": 4.9 }, { "id": "a669d373699062eae42d", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.6829999999999999, "raw_score": 68.3, "raw_scale": 100, "source_record": "gpt-6-1-terra|loop_truncated_tools_agent|pass-1", "source_model_id": "gpt-6-1-terra", "n_samples": 959, "reported_error": 5.2 }, { "id": "ff5e3f7b7475b2372fcb", "model": "Gemini 3.7 Flash", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.6779999999999999, "raw_score": 67.8, "raw_scale": 100, "source_record": "gemini-3.7-flash|loop_truncated_tools_agent|pass-1", "source_model_id": "gemini-3.7-flash", "n_samples": 952, "reported_error": 5.1 }, { "id": "2692b7b78d61b2f4373c", "model": "Muse Wonder 1.0", "organization": "Meta", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.6609999999999999, "raw_score": 66.1, "raw_scale": 100, "source_record": "muse-wonder-1-0-max|loop_truncated_tools_agent|pass-1", "source_model_id": "muse-wonder-1-0-max", "n_samples": 960, "reported_error": 5.3 }, { "id": "75e16388320ff1f78ac3", "model": "Opus 5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.6579999999999999, "raw_score": 65.8, "raw_scale": 100, "source_record": "claude-opus-5|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-opus-5", "n_samples": 956, "reported_error": 5.1 }, { "id": "3eee280bacafc0cbc92e", "model": "Grok 4.6", "organization": "xAI", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.653, "raw_score": 65.3, "raw_scale": 100, "source_record": "grok-4-6-xhigh|loop_truncated_tools_agent|pass-1", "source_model_id": "grok-4-6-xhigh", "n_samples": 960, "reported_error": 5.2 }, { "id": "04bc83cceb6766483fb8", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.647, "raw_score": 64.7, "raw_scale": 100, "source_record": "gpt-6-astra-max|loop_truncated_tools_agent|pass-1", "source_model_id": "gpt-6-astra-max", "n_samples": 959, "reported_error": 5.6 }, { "id": "43daad8656da9a867777", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.643, "raw_score": 64.3, "raw_scale": 100, "source_record": "gemini-3.8-flash|loop_truncated_tools_agent|pass-1", "source_model_id": "gemini-3.8-flash", "n_samples": 956, "reported_error": 5.2 }, { "id": "30acfc270b393d513cbf", "model": "Fable 5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.636, "raw_score": 63.6, "raw_scale": 100, "source_record": "claude-fable-5|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-fable-5", "n_samples": 959, "reported_error": 5.3 }, { "id": "4feea1eb33ce96c2ba95", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.633, "raw_score": 63.3, "raw_scale": 100, "source_record": "qwen-3-8-max-xhigh|loop_truncated_tools_agent|pass-1", "source_model_id": "qwen-3-8-max-xhigh", "n_samples": 951, "reported_error": 4.9 }, { "id": "305154f2d5b1974eaf40", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.6, "raw_score": 60.0, "raw_scale": 100, "source_record": "gpt-6-1-sol|loop_truncated_tools_agent|pass-1", "source_model_id": "gpt-6-1-sol", "n_samples": 954, "reported_error": 5.6 }, { "id": "b39a1507985a1dd04db6", "model": "Fable 5.1", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.597, "raw_score": 59.7, "raw_scale": 100, "source_record": "claude-fable-5.1-high|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-fable-5.1-high", "n_samples": 954, "reported_error": 5.6 }, { "id": "0926508a6e4a7e6024bd", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.586, "raw_score": 58.6, "raw_scale": 100, "source_record": "muse-spark-1-3|loop_truncated_tools_agent|pass-1", "source_model_id": "muse-spark-1-3", "n_samples": 959, "reported_error": 5.5 }, { "id": "973f96f0e09d1c65fe5f", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.5820000000000001, "raw_score": 58.2, "raw_scale": 100, "source_record": "gpt-5.6-terra-max|loop_truncated_tools_agent|pass-1", "source_model_id": "gpt-5.6-terra-max", "n_samples": 958, "reported_error": 5.3 }, { "id": "7bc56c11c5e7cc2848a6", "model": "MiMo V2.6 Flash RL", "organization": "Xiaomi", "configuration": "Agent: Loop; reasoning=auto; pass-1", "score": 0.574, "raw_score": 57.4, "raw_scale": 100, "source_record": "mimo-v2.6-flash-rl|loop_truncated_tools_agent|pass-1", "source_model_id": "mimo-v2.6-flash-rl", "n_samples": 940, "reported_error": 5.1 }, { "id": "bdb9fd5677fe1fd66163", "model": "GLM-5.3", "organization": "Zhipu", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.5660000000000001, "raw_score": 56.6, "raw_scale": 100, "source_record": "glm-5-3|loop_truncated_tools_agent|pass-1", "source_model_id": "glm-5-3", "n_samples": 960, "reported_error": 5.1 }, { "id": "a9f263749e5610e47fd3", "model": "Grok 4.5", "organization": "xAI", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.562, "raw_score": 56.2, "raw_scale": 100, "source_record": "grok-4-5|loop_truncated_tools_agent|pass-1", "source_model_id": "grok-4-5", "n_samples": 960, "reported_error": 5.4 }, { "id": "86c37f738ffe3a53ae20", "model": "Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.561, "raw_score": 56.1, "raw_scale": 100, "source_record": "claude-haiku-5-5|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-haiku-5-5", "n_samples": 959, "reported_error": 5.2 }, { "id": "33a0b34b8f15fcf26f6e", "model": "DeepSeek-V4-Flash", "organization": "DeepSeek", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.5529999999999999, "raw_score": 55.3, "raw_scale": 100, "source_record": "deepseek-v4-flash|loop_truncated_tools_agent|pass-1", "source_model_id": "deepseek-v4-flash", "n_samples": 829, "reported_error": 5.6 }, { "id": "2513190124316c9ae3f2", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.551, "raw_score": 55.1, "raw_scale": 100, "source_record": "gpt-5.5-xhigh|loop_truncated_tools_agent|pass-1", "source_model_id": "gpt-5.5-xhigh", "n_samples": 959, "reported_error": 5.5 }, { "id": "a6d1e53134e8741c0798", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.546, "raw_score": 54.6, "raw_scale": 100, "source_record": "grok-4.7|loop_truncated_tools_agent|pass-1", "source_model_id": "grok-4.7", "n_samples": 960, "reported_error": 5.1 }, { "id": "bd4c2653b486e5c3f550", "model": "Sonnet 5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.545, "raw_score": 54.5, "raw_scale": 100, "source_record": "claude-sonnet-5-max|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-sonnet-5-max", "n_samples": 958, "reported_error": 5.5 }, { "id": "2f5725090719c2288ee4", "model": "MiMo V2.6 Pro", "organization": "Xiaomi", "configuration": "Agent: Loop; reasoning=auto; pass-1", "score": 0.544, "raw_score": 54.4, "raw_scale": 100, "source_record": "mimo-v2.6-pro|loop_truncated_tools_agent|pass-1", "source_model_id": "mimo-v2.6-pro", "n_samples": 960, "reported_error": 5.2 }, { "id": "6e0a018a281e35f27bc7", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.5429999999999999, "raw_score": 54.3, "raw_scale": 100, "source_record": "gpt-6-sol|loop_truncated_tools_agent|pass-1", "source_model_id": "gpt-6-sol", "n_samples": 951, "reported_error": 5.6 }, { "id": "84fc7087db90c2b4d91e", "model": "MiniMax-M3.1-Flash", "organization": "MiniMax", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.539, "raw_score": 53.9, "raw_scale": 100, "source_record": "minimax-m3p1-flash|loop_truncated_tools_agent|pass-1", "source_model_id": "minimax-m3p1-flash", "n_samples": 960, "reported_error": 5.1 }, { "id": "828876039a9bb7080927", "model": "GLM-5.3-Flash", "organization": "Zhipu", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.528, "raw_score": 52.8, "raw_scale": 100, "source_record": "glm-5-3-flash|loop_truncated_tools_agent|pass-1", "source_model_id": "glm-5-3-flash", "n_samples": 957, "reported_error": 5.3 }, { "id": "8a3f8c683f3c89a92f4c", "model": "Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=medium; pass-1", "score": 0.525, "raw_score": 52.5, "raw_scale": 100, "source_record": "opus-5.5|loop_truncated_tools_agent|pass-1", "source_model_id": "opus-5.5", "n_samples": 943, "reported_error": 5.6 }, { "id": "9eb3d736d656826744d7", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.524, "raw_score": 52.4, "raw_scale": 100, "source_record": "responses/gpt-5.4|loop_truncated_tools_agent|pass-1", "source_model_id": "responses/gpt-5.4", "n_samples": 958, "reported_error": 5.3 }, { "id": "64c5c0a53338d522201f", "model": "Mistral Large 4", "organization": "Mistral", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.516, "raw_score": 51.6, "raw_scale": 100, "source_record": "mistral-large-4|loop_truncated_tools_agent|pass-1", "source_model_id": "mistral-large-4", "n_samples": 955, "reported_error": 5.2 }, { "id": "66d7ccbb315e59fda407", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.515, "raw_score": 51.5, "raw_scale": 100, "source_record": "gpt-5-6-sol-max-pro|loop_truncated_tools_agent|pass-1", "source_model_id": "gpt-5-6-sol-max-pro", "n_samples": 958, "reported_error": 5.5 }, { "id": "e2fbf990ea47ad067904", "model": "Mercury 3", "organization": "Inception", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.502, "raw_score": 50.2, "raw_scale": 100, "source_record": "inception/mercury-3|loop_truncated_tools_agent|pass-1", "source_model_id": "inception/mercury-3", "n_samples": 855, "reported_error": 5.6 }, { "id": "b277c015c9a4ebdb385e", "model": "Opus 4.7", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.49200000000000005, "raw_score": 49.2, "raw_scale": 100, "source_record": "claude-opus-4-7|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-opus-4-7", "n_samples": 960, "reported_error": 5.4 }, { "id": "baf710681f4577f6805e", "model": "Opus 4.8", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.489, "raw_score": 48.9, "raw_scale": 100, "source_record": "claude-opus-4-8|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-opus-4-8", "n_samples": 961, "reported_error": 5.5 }, { "id": "4fa100fde77bbe32cbdb", "model": "Beam", "organization": "Reflection", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.488, "raw_score": 48.8, "raw_scale": 100, "source_record": "beam-1-0-max|loop_truncated_tools_agent|pass-1", "source_model_id": "beam-1-0-max", "n_samples": 952, "reported_error": 5.1 }, { "id": "637ecc0bdd9e18bbb551", "model": "Kimi K3", "organization": "Kimi", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.478, "raw_score": 47.8, "raw_scale": 100, "source_record": "kimi-k3|loop_truncated_tools_agent|pass-1", "source_model_id": "kimi-k3", "n_samples": 960, "reported_error": 5.2 }, { "id": "7afe0356399b44a5bc40", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.47600000000000003, "raw_score": 47.6, "raw_scale": 100, "source_record": "muse-spark-1-3-max|loop_truncated_tools_agent|pass-1", "source_model_id": "muse-spark-1-3-max", "n_samples": 960, "reported_error": 5.7 }, { "id": "c56d2314a7aef655b64e", "model": "Qwen 3.8 27B", "organization": "Alibaba", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.475, "raw_score": 47.5, "raw_scale": 100, "source_record": "qwen-3-8-27b|loop_truncated_tools_agent|pass-1", "source_model_id": "qwen-3-8-27b", "n_samples": 957, "reported_error": 4.9 }, { "id": "3b414d5775abef47f333", "model": "DeepSeek-V4-Pro-0813", "organization": "DeepSeek", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.473, "raw_score": 47.3, "raw_scale": 100, "source_record": "deepseek-v4-pro-08-13|loop_truncated_tools_agent|pass-1", "source_model_id": "deepseek-v4-pro-08-13", "n_samples": 849, "reported_error": 5.6 }, { "id": "25a4eefc0fee040518d0", "model": "Gemini 3.6 Flash", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.469, "raw_score": 46.9, "raw_scale": 100, "source_record": "gemini-3.6-flash|loop_truncated_tools_agent|pass-1", "source_model_id": "gemini-3.6-flash", "n_samples": 956, "reported_error": 5.2 }, { "id": "28f5a27a74f7d92ac9d0", "model": "Opus 4.6", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.46299999999999997, "raw_score": 46.3, "raw_scale": 100, "source_record": "claude-opus-4-6-max|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-opus-4-6-max", "n_samples": 955, "reported_error": 5.5 }, { "id": "7cf9a8368a86066fb0fa", "model": "Muse Spark 1.2", "organization": "Meta", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.45399999999999996, "raw_score": 45.4, "raw_scale": 100, "source_record": "muse-spark-1-2|loop_truncated_tools_agent|pass-1", "source_model_id": "muse-spark-1-2", "n_samples": 960, "reported_error": 5 }, { "id": "cb1745a4ff0a6e8ec3e6", "model": "GLM-5.2", "organization": "Zhipu", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.452, "raw_score": 45.2, "raw_scale": 100, "source_record": "glm-5-2|loop_truncated_tools_agent|pass-1", "source_model_id": "glm-5-2", "n_samples": 846, "reported_error": 5.7 }, { "id": "ef03bad5f9b455d0e6af", "model": "Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=medium; pass-1", "score": 0.446, "raw_score": 44.6, "raw_scale": 100, "source_record": "claude-sonnet-5-5-medium|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-sonnet-5-5-medium", "n_samples": 955, "reported_error": 5.6 }, { "id": "11f06931eed8154c93fe", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.44299999999999995, "raw_score": 44.3, "raw_scale": 100, "source_record": "gpt-6-luna|loop_truncated_tools_agent|pass-1", "source_model_id": "gpt-6-luna", "n_samples": 946, "reported_error": 5.5 }, { "id": "8965bed6f79a3d8dfae2", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.43, "raw_score": 43.0, "raw_scale": 100, "source_record": "gpt-5.6-luna-max|loop_truncated_tools_agent|pass-1", "source_model_id": "gpt-5.6-luna-max", "n_samples": 945, "reported_error": 5.4 }, { "id": "aa915bcd8caa49b1784d", "model": "Sonnet 4.6", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.43, "raw_score": 43.0, "raw_scale": 100, "source_record": "claude-sonnet-4-6-high|loop_truncated_tools_agent|pass-1", "source_model_id": "claude-sonnet-4-6-high", "n_samples": 949, "reported_error": 5.5 }, { "id": "2080d99faa04179fb1dd", "model": "GLM-5.1", "organization": "Zhipu", "configuration": "Agent: Loop; pass-1", "score": 0.409, "raw_score": 40.9, "raw_scale": 100, "source_record": "glm-5-1|loop_truncated_tools_agent|pass-1", "source_model_id": "glm-5-1", "n_samples": 856, "reported_error": 5.4 }, { "id": "ceac9b15740441fa064d", "model": "DeepSeek-V4.1-Flash", "organization": "DeepSeek", "configuration": "Agent: Loop; reasoning=max; pass-1", "score": 0.395, "raw_score": 39.5, "raw_scale": 100, "source_record": "deepseek-v4-1-flash|loop_truncated_tools_agent|pass-1", "source_model_id": "deepseek-v4-1-flash", "n_samples": 960, "reported_error": 4.9 }, { "id": "043481df5035757ab70b", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.377, "raw_score": 37.7, "raw_scale": 100, "source_record": "minimax-m3|loop_truncated_tools_agent|pass-1", "source_model_id": "minimax-m3", "n_samples": 951, "reported_error": 4.9 }, { "id": "5ee1e25b70cbbfca67a1", "model": "Kimi K2.7 Code", "organization": "Kimi", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.376, "raw_score": 37.6, "raw_scale": 100, "source_record": "kimi-k2-7-code-high|loop_truncated_tools_agent|pass-1", "source_model_id": "kimi-k2-7-code-high", "n_samples": 950, "reported_error": 4.8 }, { "id": "aec84c7d1751046e49d7", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.35100000000000003, "raw_score": 35.1, "raw_scale": 100, "source_record": "gemini-3.1-pro-preview-high|loop_truncated_tools_agent|pass-1", "source_model_id": "gemini-3.1-pro-preview-high", "n_samples": 960, "reported_error": 5 }, { "id": "ec50152f6923bf46c697", "model": "Inkling", "organization": "Thinking Machines", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.33799999999999997, "raw_score": 33.8, "raw_scale": 100, "source_record": "inkling|loop_truncated_tools_agent|pass-1", "source_model_id": "inkling", "n_samples": 946, "reported_error": 4.9 }, { "id": "d64d798327dec182497c", "model": "Muse Spark Open", "organization": "Meta", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.33399999999999996, "raw_score": 33.4, "raw_scale": 100, "source_record": "muse-spark-open|loop_truncated_tools_agent|pass-1", "source_model_id": "muse-spark-open", "n_samples": 956, "reported_error": 5.3 }, { "id": "e88d3ce3c19491909afb", "model": "Muse Spark 1.1", "organization": "Meta", "configuration": "Agent: Loop; reasoning=xhigh; pass-1", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "muse-spark-1-1|loop_truncated_tools_agent|pass-1", "source_model_id": "muse-spark-1-1", "n_samples": 819, "reported_error": 5.1 }, { "id": "2e506b62969bf868824b", "model": "Nemotron 3.5 Super VL", "organization": "NVIDIA", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.295, "raw_score": 29.5, "raw_scale": 100, "source_record": "nemotron-3-5-super-vl|loop_truncated_tools_agent|pass-1", "source_model_id": "nemotron-3-5-super-vl", "n_samples": 960, "reported_error": 4.9 }, { "id": "c9bcf09822605b500349", "model": "Gemini 3.5 Flash Lite", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.293, "raw_score": 29.3, "raw_scale": 100, "source_record": "gemini-3-5-flash-lite|loop_truncated_tools_agent|pass-1", "source_model_id": "gemini-3-5-flash-lite", "n_samples": 954, "reported_error": 4.9 }, { "id": "35371ff985658e1afe0d", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.275, "raw_score": 27.5, "raw_scale": 100, "source_record": "gemini-3.5-flash|loop_truncated_tools_agent|pass-1", "source_model_id": "gemini-3.5-flash", "n_samples": 959, "reported_error": 5 }, { "id": "1ca531db32f73667345a", "model": "Nemotron 3.5 Bolt", "organization": "NVIDIA", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.27399999999999997, "raw_score": 27.4, "raw_scale": 100, "source_record": "nemotron-3-5-bolt|loop_truncated_tools_agent|pass-1", "source_model_id": "nemotron-3-5-bolt", "n_samples": 960, "reported_error": 4.6 }, { "id": "996a59c25c76c2c29a39", "model": "Qwen3.5", "organization": "Alibaba", "configuration": "Agent: Loop; pass-1", "score": 0.249, "raw_score": 24.9, "raw_scale": 100, "source_record": "Qwen3.5-397B-A17B|loop_truncated_tools_agent|pass-1", "source_model_id": "Qwen3.5-397B-A17B", "n_samples": 960, "reported_error": 4.7 }, { "id": "f1d4eae4f7a5e63195a9", "model": "Nemotron 3 Ultra", "organization": "NVIDIA", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.22699999999999998, "raw_score": 22.7, "raw_scale": 100, "source_record": "nemotron-3-ultra-nvfp4|loop_truncated_tools_agent|pass-1", "source_model_id": "nemotron-3-ultra-nvfp4", "n_samples": 887, "reported_error": 4.5 }, { "id": "272c8d86edd39e16aef4", "model": "DeepSeek-V3.2", "organization": "DeepSeek", "configuration": "Agent: Loop; pass-1", "score": 0.213, "raw_score": 21.3, "raw_scale": 100, "source_record": "deepseek-ai/DeepSeek-V3.2|loop_truncated_tools_agent|pass-1", "source_model_id": "deepseek-ai/DeepSeek-V3.2", "n_samples": 952, "reported_error": 4.1 }, { "id": "799486310f3b679ceea2", "model": "GPT-OSS-120B", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=high; pass-1", "score": 0.044000000000000004, "raw_score": 4.4, "raw_scale": 100, "source_record": "openai/gpt-oss-120b|loop_truncated_tools_agent|pass-1", "source_model_id": "openai/gpt-oss-120b", "n_samples": 954, "reported_error": 2 } ] }, { "id": "apex-v1-mercor", "name": "APEX-1", "group": "专业工作", "unit": "score", "metric": "mean-score", "protocol": { "en": "Mercor global leaderboard; mean rubric score on 0–100. Each model/reasoning/harness configuration remains separate.", "zh": "Mercor 总榜；0–100 的量规平均分。各模型、推理与 Agent 配置分别记录。" }, "source": { "name": "Mercor", "benchmark_creator": "Mercor", "url": "https://www.mercor.com/apex/", "data_url": "https://www.mercor.com/apex/", "sha256": "9a1f8200f24bae14929d7d2cf9129f85b338b131811f0f93bfc86c8aff535edc", "retrieved_at": "2026-11-02T03:34:24.031268+00:00", "selector": "__NEXT_DATA__.props.pageProps.benchmarks[benchmarkId=apex-v1].globalLeaderboard; mean-score; harnessScores", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "6fdadfb15feea40ec305", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 81.1, "raw_score": 81.1, "raw_scale": 1, "source_record": "gemini-4-argon|singleshot_agent|mean-score", "source_model_id": "gemini-4-argon", "n_samples": 1600, "reported_error": 2 }, { "id": "d020e6885e567cb90625", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 80.0, "raw_score": 80.0, "raw_scale": 1, "source_record": "gemini-4-carbon|singleshot_agent|mean-score", "source_model_id": "gemini-4-carbon", "n_samples": 1600, "reported_error": 2.1 }, { "id": "5902d9d8d9284e1cfdd5", "model": "Muse Spark 1.2", "organization": "Meta", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 75.6, "raw_score": 75.6, "raw_scale": 1, "source_record": "muse-spark-1-2|singleshot_agent|mean-score", "source_model_id": "muse-spark-1-2", "n_samples": 1593, "reported_error": 2.3 }, { "id": "1802bf0d39521d0d141a", "model": "Muse Wonder 1.0", "organization": "Meta", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 75.1, "raw_score": 75.1, "raw_scale": 1, "source_record": "muse-wonder-1-0-max|singleshot_agent|mean-score", "source_model_id": "muse-wonder-1-0-max", "n_samples": 1600, "reported_error": 2.2 }, { "id": "2defe862d54f01163853", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 73.9, "raw_score": 73.9, "raw_scale": 1, "source_record": "muse-spark-1-3|singleshot_agent|mean-score", "source_model_id": "muse-spark-1-3", "n_samples": 1600, "reported_error": 2.3 }, { "id": "83ad09be1a22aab49d09", "model": "Muse Spark Open", "organization": "Meta", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 73.9, "raw_score": 73.9, "raw_scale": 1, "source_record": "muse-spark-open|singleshot_agent|mean-score", "source_model_id": "muse-spark-open", "n_samples": 1600, "reported_error": 2.3 }, { "id": "93b2e6fb0a376430234f", "model": "Fable 5.1", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 73.8, "raw_score": 73.8, "raw_scale": 1, "source_record": "claude-fable-5.1|singleshot_agent|mean-score", "source_model_id": "claude-fable-5.1", "n_samples": 1596, "reported_error": 2.7 }, { "id": "71372d2115edd21a986f", "model": "Muse Spark 1.1", "organization": "Meta", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 73.8, "raw_score": 73.8, "raw_scale": 1, "source_record": "muse-spark-1-1|singleshot_agent|mean-score", "source_model_id": "muse-spark-1-1", "n_samples": 1591, "reported_error": 2.2 }, { "id": "1d226cb38ed9e16ccf04", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 73.1, "raw_score": 73.1, "raw_scale": 1, "source_record": "gpt-5-6-sol-max-pro|singleshot_agent|mean-score", "source_model_id": "gpt-5-6-sol-max-pro", "n_samples": 1600, "reported_error": 2.4 }, { "id": "15cf3f9741e06598b51f", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 73.0, "raw_score": 73.0, "raw_scale": 1, "source_record": "muse-spark-1-3-max|singleshot_agent|mean-score", "source_model_id": "muse-spark-1-3-max", "n_samples": 1600, "reported_error": 2.3 }, { "id": "eda9d43681948aa71e88", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 69.5, "raw_score": 69.5, "raw_scale": 1, "source_record": "gpt-5.6-terra-max|singleshot_agent|mean-score", "source_model_id": "gpt-5.6-terra-max", "n_samples": 1598, "reported_error": 2.4 }, { "id": "1b9edc4cd2d8dcec31a3", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 69.3, "raw_score": 69.3, "raw_scale": 1, "source_record": "gpt-6-astra|singleshot_agent|mean-score", "source_model_id": "gpt-6-astra", "n_samples": 1580, "reported_error": 2.6 }, { "id": "4b4ad15e7266e2d37279", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 69.3, "raw_score": 69.3, "raw_scale": 1, "source_record": "gpt-6-1-sol|singleshot_agent|mean-score", "source_model_id": "gpt-6-1-sol", "n_samples": 1436, "reported_error": 2.6 }, { "id": "4f183659d708e949305f", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 68.8, "raw_score": 68.8, "raw_scale": 1, "source_record": "gemini-3.8-flash|singleshot_agent|mean-score", "source_model_id": "gemini-3.8-flash", "n_samples": 1599, "reported_error": 2.7 }, { "id": "acfe606f8070af78d021", "model": "Opus 5", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 68.6, "raw_score": 68.6, "raw_scale": 1, "source_record": "claude-opus-5-xhigh|singleshot_agent|mean-score", "source_model_id": "claude-opus-5-xhigh", "n_samples": 1595, "reported_error": 2.9 }, { "id": "1b4ac9516e182beb059b", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 68.4, "raw_score": 68.4, "raw_scale": 1, "source_record": "gpt-5.5-xhigh|singleshot_agent|mean-score", "source_model_id": "gpt-5.5-xhigh", "n_samples": 1600, "reported_error": 2.4 }, { "id": "1d1cef947611f92187a7", "model": "Kimi K3", "organization": "Kimi", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 68.4, "raw_score": 68.4, "raw_scale": 1, "source_record": "kimi-k3|singleshot_agent|mean-score", "source_model_id": "kimi-k3", "n_samples": 1600, "reported_error": 2.3 }, { "id": "f10140447f55e7d93f6f", "model": "DeepSeek-V4.1-Flash-1017", "organization": "DeepSeek", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 68.3, "raw_score": 68.3, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-1017|singleshot_agent|mean-score", "source_model_id": "deepseek-v4-1-flash-1017", "n_samples": 1600, "reported_error": 2.5 }, { "id": "dcbba3c30694035e7c45", "model": "Grok 4.6", "organization": "xAI", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 67.7, "raw_score": 67.7, "raw_scale": 1, "source_record": "grok-4-6-xhigh|singleshot_agent|mean-score", "source_model_id": "grok-4-6-xhigh", "n_samples": 1600, "reported_error": 2.5 }, { "id": "907f7022e4a98b1eb498", "model": "Gemini 3.7 Flash", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 67.0, "raw_score": 67.0, "raw_scale": 1, "source_record": "gemini-3.7-flash|singleshot_agent|mean-score", "source_model_id": "gemini-3.7-flash", "n_samples": 1600, "reported_error": 2.6 }, { "id": "993659745b77662056fb", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 66.6, "raw_score": 66.6, "raw_scale": 1, "source_record": "gpt-5.6-luna-max|singleshot_agent|mean-score", "source_model_id": "gpt-5.6-luna-max", "n_samples": 1586, "reported_error": 2.5 }, { "id": "93c77fbd2a626e430bc4", "model": "DeepSeek-V4-Pro-0813", "organization": "DeepSeek", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 66.0, "raw_score": 66.0, "raw_scale": 1, "source_record": "deepseek-v4-pro-08-13|singleshot_agent|mean-score", "source_model_id": "deepseek-v4-pro-08-13", "n_samples": 1600, "reported_error": 2.5 }, { "id": "435b22b3b874ce0df652", "model": "Fable 5", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 66.0, "raw_score": 66.0, "raw_scale": 1, "source_record": "claude-fable-5|singleshot_agent|mean-score", "source_model_id": "claude-fable-5", "n_samples": 1600, "reported_error": 2.9 }, { "id": "5e1d274191c132053c87", "model": "Gemini 3.6 Flash", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 66.0, "raw_score": 66.0, "raw_scale": 1, "source_record": "gemini-3.6-flash|singleshot_agent|mean-score", "source_model_id": "gemini-3.6-flash", "n_samples": 1600, "reported_error": 2.5 }, { "id": "de5c147c9c3d71eaf058", "model": "GPT-5.3 Codex", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 65.9, "raw_score": 65.9, "raw_scale": 1, "source_record": "gpt-5.3-codex-high|singleshot_agent|mean-score", "source_model_id": "gpt-5.3-codex-high", "n_samples": 1192, "reported_error": 2.5 }, { "id": "2cb3aa4b8763a1717c72", "model": "Opus 4.6", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 65.8, "raw_score": 65.8, "raw_scale": 1, "source_record": "claude-opus-4-6-max|singleshot_agent|mean-score", "source_model_id": "claude-opus-4-6-max", "n_samples": 2000, "reported_error": 2.6 }, { "id": "926767f9a5c051b658cc", "model": "Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 65.8, "raw_score": 65.8, "raw_scale": 1, "source_record": "claude-opus-5-5|singleshot_agent|mean-score", "source_model_id": "claude-opus-5-5", "n_samples": 1600, "reported_error": 3.2 }, { "id": "3cb66fc49d2328b67cbb", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 65.6, "raw_score": 65.6, "raw_scale": 1, "source_record": "responses/gpt-5.4|singleshot_agent|mean-score", "source_model_id": "responses/gpt-5.4", "n_samples": 1600, "reported_error": 2.4 }, { "id": "3fb1ae2625137f31b7e8", "model": "DeepSeek-V4.1-Flash", "organization": "DeepSeek", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 65.5, "raw_score": 65.5, "raw_scale": 1, "source_record": "deepseek-v4-1-flash|singleshot_agent|mean-score", "source_model_id": "deepseek-v4-1-flash", "n_samples": 1600, "reported_error": 2.6 }, { "id": "d0f33a3ca9a49f90746a", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 65.4, "raw_score": 65.4, "raw_scale": 1, "source_record": "grok-4.7|singleshot_agent|mean-score", "source_model_id": "grok-4.7", "n_samples": 1591, "reported_error": 2.6 }, { "id": "b482e550fa3cc2f5c572", "model": "Opus 4.6", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 65.3, "raw_score": 65.3, "raw_scale": 1, "source_record": "claude-opus-4-6-high|singleshot_agent|mean-score", "source_model_id": "claude-opus-4-6-high", "n_samples": 2000, "reported_error": 2.6 }, { "id": "cf0d9ba68e6a73e44b52", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 65.3, "raw_score": 65.3, "raw_scale": 1, "source_record": "qwen-3-8-max-xhigh|singleshot_agent|mean-score", "source_model_id": "qwen-3-8-max-xhigh", "n_samples": 1559, "reported_error": 2.5 }, { "id": "827595847f79cb73f9fe", "model": "MiniMax-M3.1-Flash", "organization": "MiniMax", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 65.0, "raw_score": 65.0, "raw_scale": 1, "source_record": "minimax-m3p1-flash|singleshot_agent|mean-score", "source_model_id": "minimax-m3p1-flash", "n_samples": 1600, "reported_error": 2.6 }, { "id": "718ed10b4c6219fdf060", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 64.7, "raw_score": 64.7, "raw_scale": 1, "source_record": "gpt-5.2-high|singleshot_agent|mean-score", "source_model_id": "gpt-5.2-high", "n_samples": 1599, "reported_error": 2.5 }, { "id": "654208e8b35c1523dc14", "model": "Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 64.7, "raw_score": 64.7, "raw_scale": 1, "source_record": "claude-haiku-5-5|singleshot_agent|mean-score", "source_model_id": "claude-haiku-5-5", "n_samples": 1600, "reported_error": 2.6 }, { "id": "e8360fe4101d454eeecb", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 64.2, "raw_score": 64.2, "raw_scale": 1, "source_record": "gpt-6-1-terra|singleshot_agent|mean-score", "source_model_id": "gpt-6-1-terra", "n_samples": 1600, "reported_error": 2.7 }, { "id": "42de296e2359329cace6", "model": "DeepSeek-V4-Flash", "organization": "DeepSeek", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 63.8, "raw_score": 63.8, "raw_scale": 1, "source_record": "deepseek-v4-flash|singleshot_agent|mean-score", "source_model_id": "deepseek-v4-flash", "n_samples": 1600, "reported_error": 2.6 }, { "id": "6bb7e5c6b996a5c32ded", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 63.6, "raw_score": 63.6, "raw_scale": 1, "source_record": "gpt-6-sol|singleshot_agent|mean-score", "source_model_id": "gpt-6-sol", "n_samples": 1588, "reported_error": 2.6 }, { "id": "1003ffadbe756f0b709b", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 63.2, "raw_score": 63.2, "raw_scale": 1, "source_record": "gpt-5.4-high|singleshot_agent|mean-score", "source_model_id": "gpt-5.4-high", "n_samples": 1600, "reported_error": 2.4 }, { "id": "97601fca459a7dad6a81", "model": "GLM-5.3", "organization": "Zhipu", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 63.0, "raw_score": 63.0, "raw_scale": 1, "source_record": "glm-5-3|singleshot_agent|mean-score", "source_model_id": "glm-5-3", "n_samples": 1600, "reported_error": 2.9 }, { "id": "55f69132b4df585f784e", "model": "Opus 4.8", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 62.7, "raw_score": 62.7, "raw_scale": 1, "source_record": "claude-opus-4-8|singleshot_agent|mean-score", "source_model_id": "claude-opus-4-8", "n_samples": 1600, "reported_error": 3 }, { "id": "3136268de7aacde8ae9c", "model": "Qwen 3.8 27B", "organization": "Alibaba", "configuration": "Agent: Single-Shot Agent; reasoning=xhigh; mean-score", "score": 62.6, "raw_score": 62.6, "raw_scale": 1, "source_record": "qwen-3-8-27b|singleshot_agent|mean-score", "source_model_id": "qwen-3-8-27b", "n_samples": 1580, "reported_error": 2.7 }, { "id": "601db509a276443dd5a7", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 62.2, "raw_score": 62.2, "raw_scale": 1, "source_record": "gemini-3.5-flash|singleshot_agent|mean-score", "source_model_id": "gemini-3.5-flash", "n_samples": 1600, "reported_error": 2.8 }, { "id": "aa1380189315111750d5", "model": "Opus 4.7", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 62.2, "raw_score": 62.2, "raw_scale": 1, "source_record": "claude-opus-4-7|singleshot_agent|mean-score", "source_model_id": "claude-opus-4-7", "n_samples": 1600, "reported_error": 2.8 }, { "id": "4120612f67f837f9b75e", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 62.0, "raw_score": 62.0, "raw_scale": 1, "source_record": "gpt-6-luna|singleshot_agent|mean-score", "source_model_id": "gpt-6-luna", "n_samples": 1600, "reported_error": 2.7 }, { "id": "b718bcae49e268fbb735", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 61.8, "raw_score": 61.8, "raw_scale": 1, "source_record": "gemini-3.1-pro-preview-high|singleshot_agent|mean-score", "source_model_id": "gemini-3.1-pro-preview-high", "n_samples": 1599, "reported_error": 2.5 }, { "id": "8360afe57a5d57a389fd", "model": "Grok 4.5", "organization": "xAI", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 61.8, "raw_score": 61.8, "raw_scale": 1, "source_record": "grok-4-5|singleshot_agent|mean-score", "source_model_id": "grok-4-5", "n_samples": 1600, "reported_error": 2.5 }, { "id": "2f8b7386ff91ce9162f8", "model": "Inkling", "organization": "Thinking Machines", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 61.6, "raw_score": 61.6, "raw_scale": 1, "source_record": "inkling|singleshot_agent|mean-score", "source_model_id": "inkling", "n_samples": 1600, "reported_error": 2.7 }, { "id": "2c7dc2f606bfbf06059c", "model": "GLM-5.2", "organization": "Zhipu", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 61.5, "raw_score": 61.5, "raw_scale": 1, "source_record": "glm-5-2|singleshot_agent|mean-score", "source_model_id": "glm-5-2", "n_samples": 1600, "reported_error": 2.5 }, { "id": "4608da3ea8de9ab0c406", "model": "Gemini 3 Flash", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 60.9, "raw_score": 60.9, "raw_scale": 1, "source_record": "gemini-3-flash-preview-high|singleshot_agent|mean-score", "source_model_id": "gemini-3-flash-preview-high", "n_samples": 1600, "reported_error": 2.3 }, { "id": "30e36d3a47a20916776b", "model": "Gemini 3 Pro", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 60.9, "raw_score": 60.9, "raw_scale": 1, "source_record": "gemini-3-pro-preview-high|singleshot_agent|mean-score", "source_model_id": "gemini-3-pro-preview-high", "n_samples": 3200, "reported_error": 2.4 }, { "id": "174eea9c30352a2c7f39", "model": "Beam", "organization": "Reflection", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 60.6, "raw_score": 60.6, "raw_scale": 1, "source_record": "beam-1-0-max|singleshot_agent|mean-score", "source_model_id": "beam-1-0-max", "n_samples": 1600, "reported_error": 2.6 }, { "id": "f5a5b0fdd38420ab5d48", "model": "o3", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 60.5, "raw_score": 60.5, "raw_scale": 1, "source_record": "o3-high|singleshot_agent|mean-score", "source_model_id": "o3-high", "n_samples": 1599, "reported_error": 2.5 }, { "id": "eb6eb572d67fcfe9fcfd", "model": "Kimi K2.7 Code", "organization": "Kimi", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 60.1, "raw_score": 60.1, "raw_scale": 1, "source_record": "kimi-k2-7-code-high|singleshot_agent|mean-score", "source_model_id": "kimi-k2-7-code-high", "n_samples": 1596, "reported_error": 2.5 }, { "id": "8ba6719342abbbef5511", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 59.8, "raw_score": 59.8, "raw_scale": 1, "source_record": "minimax-m3|singleshot_agent|mean-score", "source_model_id": "minimax-m3", "n_samples": 1600, "reported_error": 2.5 }, { "id": "c20f573f68c21d58db41", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 59.7, "raw_score": 59.7, "raw_scale": 1, "source_record": "deepseek-v4-pro|singleshot_agent|mean-score", "source_model_id": "deepseek-v4-pro", "n_samples": 1594, "reported_error": 2.7 }, { "id": "a7b82f286afa0ced88a9", "model": "Mistral Large 4", "organization": "Mistral", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 59.6, "raw_score": 59.6, "raw_scale": 1, "source_record": "mistral-large-4|singleshot_agent|mean-score", "source_model_id": "mistral-large-4", "n_samples": 1593, "reported_error": 2.5 }, { "id": "e4d2ec2ccc117723d8da", "model": "GLM-5.3-Flash", "organization": "Zhipu", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 58.7, "raw_score": 58.7, "raw_scale": 1, "source_record": "glm-5-3-flash|singleshot_agent|mean-score", "source_model_id": "glm-5-3-flash", "n_samples": 1600, "reported_error": 3.1 }, { "id": "b78076266c9e43d9f066", "model": "Nemotron 3.5 Super VL", "organization": "NVIDIA", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 57.6, "raw_score": 57.6, "raw_scale": 1, "source_record": "nemotron-3-5-super-vl|singleshot_agent|mean-score", "source_model_id": "nemotron-3-5-super-vl", "n_samples": 1600, "reported_error": 2.6 }, { "id": "f3e98a6ce423027264ee", "model": "Grok 4.1", "organization": "xAI", "configuration": "Agent: Single-Shot Agent; reasoning=fast; mean-score", "score": 57.5, "raw_score": 57.5, "raw_scale": 1, "source_record": "grok-4-1-fast-reasoning|singleshot_agent|mean-score", "source_model_id": "grok-4-1-fast-reasoning", "n_samples": 1600, "reported_error": 2.6 }, { "id": "b972182895c3fe4705ba", "model": "Opus 4.5", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 57.3, "raw_score": 57.3, "raw_scale": 1, "source_record": "claude-opus-4-5-high|singleshot_agent|mean-score", "source_model_id": "claude-opus-4-5-high", "n_samples": 3200, "reported_error": 2.8 }, { "id": "815adf5fe1f13501cb33", "model": "Grok 4", "organization": "xAI", "configuration": "Agent: Single-Shot Agent; mean-score", "score": 57.1, "raw_score": 57.1, "raw_scale": 1, "source_record": "grok-4|singleshot_agent|mean-score", "source_model_id": "grok-4", "n_samples": 1592, "reported_error": 2.7 }, { "id": "5332c01dd4ec8ce42468", "model": "Nemotron 3 Ultra", "organization": "NVIDIA", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 56.1, "raw_score": 56.1, "raw_scale": 1, "source_record": "nemotron-3-ultra-nvfp4|singleshot_agent|mean-score", "source_model_id": "nemotron-3-ultra-nvfp4", "n_samples": 1600, "reported_error": 2.6 }, { "id": "ff28adbafe650a7c1ff1", "model": "Sonnet 4.6", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 55.1, "raw_score": 55.1, "raw_scale": 1, "source_record": "claude-sonnet-4-6-high|singleshot_agent|mean-score", "source_model_id": "claude-sonnet-4-6-high", "n_samples": 1600, "reported_error": 3.3 }, { "id": "1da6b3e42d0c7d781950", "model": "Gemini 3.5 Flash Lite", "organization": "Google", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 54.7, "raw_score": 54.7, "raw_scale": 1, "source_record": "gemini-3-5-flash-lite|singleshot_agent|mean-score", "source_model_id": "gemini-3-5-flash-lite", "n_samples": 1600, "reported_error": 2.5 }, { "id": "e48d40539023f7fbda7e", "model": "Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 53.9, "raw_score": 53.9, "raw_scale": 1, "source_record": "claude-sonnet-5-5|singleshot_agent|mean-score", "source_model_id": "claude-sonnet-5-5", "n_samples": 1598, "reported_error": 3.7 }, { "id": "3cec0166a2a5c03c63c2", "model": "Nemotron 3.5 Bolt", "organization": "NVIDIA", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 52.7, "raw_score": 52.7, "raw_scale": 1, "source_record": "nemotron-3-5-bolt|singleshot_agent|mean-score", "source_model_id": "nemotron-3-5-bolt", "n_samples": 1600, "reported_error": 2.5 }, { "id": "eb66aeb83ed3b2cd2fa1", "model": "GLM-5.1", "organization": "Zhipu", "configuration": "Agent: Single-Shot Agent; mean-score", "score": 52.4, "raw_score": 52.4, "raw_scale": 1, "source_record": "glm-5-1|singleshot_agent|mean-score", "source_model_id": "glm-5-1", "n_samples": 1576, "reported_error": 2.5 }, { "id": "9ecb02e1bcc9cce7f8b4", "model": "Kimi K2", "organization": "Kimi", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 52.4, "raw_score": 52.4, "raw_scale": 1, "source_record": "kimi-k2-thinking-high|singleshot_agent|mean-score", "source_model_id": "kimi-k2-thinking-high", "n_samples": 1517, "reported_error": 2.5 }, { "id": "448162b7916b9610a79f", "model": "DeepSeek-V3.2", "organization": "DeepSeek", "configuration": "Agent: Single-Shot Agent; mean-score", "score": 51.5, "raw_score": 51.5, "raw_scale": 1, "source_record": "deepseek-ai/DeepSeek-V3.2|singleshot_agent|mean-score", "source_model_id": "deepseek-ai/DeepSeek-V3.2", "n_samples": 1599, "reported_error": 2.7 }, { "id": "fea5ddbb01fdd2a23746", "model": "Qwen3.5", "organization": "Alibaba", "configuration": "Agent: Single-Shot Agent; mean-score", "score": 50.3, "raw_score": 50.3, "raw_scale": 1, "source_record": "Qwen3.5-397B-A17B|singleshot_agent|mean-score", "source_model_id": "Qwen3.5-397B-A17B", "n_samples": 1600, "reported_error": 2.5 }, { "id": "9c9b5b262e19d83cfa36", "model": "Sonnet 5", "organization": "Anthropic", "configuration": "Agent: Single-Shot Agent; reasoning=max; mean-score", "score": 49.6, "raw_score": 49.6, "raw_scale": 1, "source_record": "claude-sonnet-5-max|singleshot_agent|mean-score", "source_model_id": "claude-sonnet-5-max", "n_samples": 1600, "reported_error": 3.4 }, { "id": "d76e9c1892817540a1c8", "model": "Gemma 4 31B", "organization": "Google", "configuration": "Agent: Single-Shot Agent; mean-score", "score": 48.1, "raw_score": 48.1, "raw_scale": 1, "source_record": "gemma-4-31B|singleshot_agent|mean-score", "source_model_id": "gemma-4-31B", "n_samples": 1574, "reported_error": 2.7 }, { "id": "5dbad273c6434644cafb", "model": "GPT-OSS-120B", "organization": "OpenAI", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 46.4, "raw_score": 46.4, "raw_scale": 1, "source_record": "openai/gpt-oss-120b|singleshot_agent|mean-score", "source_model_id": "openai/gpt-oss-120b", "n_samples": 1600, "reported_error": 2.7 }, { "id": "858aa0b616b0bc34759f", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "Agent: Single-Shot Agent; reasoning=high; mean-score", "score": 45.8, "raw_score": 45.8, "raw_scale": 1, "source_record": "minimax-m2-7|singleshot_agent|mean-score", "source_model_id": "minimax-m2-7", "n_samples": 1600, "reported_error": 2.7 } ], "score_max": 100 }, { "id": "apex-swe-mercor", "name": "APEX-SWE", "group": "编程", "unit": "%", "metric": "pass-1", "protocol": { "en": "Mercor global leaderboard; complete task pass@1. Each model/reasoning/harness configuration remains separate.", "zh": "Mercor 总榜；完整任务 pass@1。各模型、推理与 Agent 配置分别记录。" }, "source": { "name": "Mercor", "benchmark_creator": "Mercor", "url": "https://www.mercor.com/apex/", "data_url": "https://www.mercor.com/apex/", "sha256": "9a1f8200f24bae14929d7d2cf9129f85b338b131811f0f93bfc86c8aff535edc", "retrieved_at": "2026-11-02T03:34:24.031268+00:00", "selector": "__NEXT_DATA__.props.pageProps.benchmarks[benchmarkId=apex-swe].globalLeaderboard; pass-1; harnessScores", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "ba118d3a81438ff217fb", "model": "Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.6759999999999999, "raw_score": 67.6, "raw_scale": 100, "source_record": "claude-opus-5-5|terminus-2|pass-1", "source_model_id": "claude-opus-5-5", "n_samples": 800, "reported_error": 5.9 }, { "id": "4e55b365a387cf7e1da9", "model": "Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.664, "raw_score": 66.4, "raw_scale": 100, "source_record": "claude-sonnet-5-5|terminus-2|pass-1", "source_model_id": "claude-sonnet-5-5", "n_samples": 800, "reported_error": 6.1 }, { "id": "67c3b1b3e3aa8a90656c", "model": "Opus 5", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.637, "raw_score": 63.7, "raw_scale": 100, "source_record": "claude-opus-5|terminus-2|pass-1", "source_model_id": "claude-opus-5", "n_samples": 800, "reported_error": 6.4 }, { "id": "63005000054841066f38", "model": "Fable 5.1", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.636, "raw_score": 63.6, "raw_scale": 100, "source_record": "claude-fable-5.1|terminus-2|pass-1", "source_model_id": "claude-fable-5.1", "n_samples": 776, "reported_error": 6.3 }, { "id": "75a3bae62fb4e7bf8c09", "model": "Fable 5", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.589, "raw_score": 58.9, "raw_scale": 100, "source_record": "claude-fable-5|terminus-2|pass-1", "source_model_id": "claude-fable-5", "n_samples": 800, "reported_error": 6.4 }, { "id": "c7795f98c84a9324ac6f", "model": "Grok 4.6", "organization": "xAI", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.5770000000000001, "raw_score": 57.7, "raw_scale": 100, "source_record": "grok-4-6-xhigh|terminus-2|pass-1", "source_model_id": "grok-4-6-xhigh", "n_samples": 1600, "reported_error": 6.2 }, { "id": "8e2da589f15ee22ac3b1", "model": "Grok 4.6", "organization": "xAI", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.564, "raw_score": 56.4, "raw_scale": 100, "source_record": "grok-4-6|terminus-2|pass-1", "source_model_id": "grok-4-6", "n_samples": 799, "reported_error": 6.2 }, { "id": "56559f962c14dafbb0e0", "model": "Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=medium; pass-1", "score": 0.5589999999999999, "raw_score": 55.9, "raw_scale": 100, "source_record": "opus-5.5|terminus-2|pass-1", "source_model_id": "opus-5.5", "n_samples": 801, "reported_error": 6.3 }, { "id": "4a7f6ad891f271e500d4", "model": "Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.551, "raw_score": 55.1, "raw_scale": 100, "source_record": "claude-haiku-5-5|terminus-2|pass-1", "source_model_id": "claude-haiku-5-5", "n_samples": 800, "reported_error": 6.5 }, { "id": "fc920c0727b90e057759", "model": "Fable 5", "organization": "Anthropic", "configuration": "Agent: Inspect; reasoning=max; pass-1", "score": 0.5483, "raw_score": 54.83, "raw_scale": 100, "source_record": "claude-fable-5|inspect|pass-1", "source_model_id": "claude-fable-5", "n_samples": 800, "reported_error": 6 }, { "id": "dea1400b742f07bffd83", "model": "Opus 5", "organization": "Anthropic", "configuration": "Agent: Inspect; reasoning=max; pass-1", "score": 0.5467, "raw_score": 54.67, "raw_scale": 100, "source_record": "claude-opus-5|inspect|pass-1", "source_model_id": "claude-opus-5", "n_samples": 800, "reported_error": 5.5 }, { "id": "4e046c9c898360882443", "model": "Muse Wonder 1.0", "organization": "Meta", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.546, "raw_score": 54.6, "raw_scale": 100, "source_record": "muse-wonder-1-0-max|terminus-2|pass-1", "source_model_id": "muse-wonder-1-0-max", "n_samples": 800, "reported_error": 6.2 }, { "id": "4039117981486cf8f2f0", "model": "MiMo V2.6 Pro", "organization": "Xiaomi", "configuration": "Agent: Terminus-2; reasoning=auto; pass-1", "score": 0.542, "raw_score": 54.2, "raw_scale": 100, "source_record": "mimo-v2.6-pro|terminus-2|pass-1", "source_model_id": "mimo-v2.6-pro", "n_samples": 793, "reported_error": 6.2 }, { "id": "3fdb142751997511d626", "model": "Grok 4.5", "organization": "xAI", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.536, "raw_score": 53.6, "raw_scale": 100, "source_record": "grok-4-5|terminus-2|pass-1", "source_model_id": "grok-4-5", "n_samples": 800, "reported_error": 6.4 }, { "id": "264084745e4ebd197bd7", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.534, "raw_score": 53.4, "raw_scale": 100, "source_record": "grok-4.7|terminus-2|pass-1", "source_model_id": "grok-4.7", "n_samples": 800, "reported_error": 6.3 }, { "id": "05a6a19bac226daab500", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.523, "raw_score": 52.3, "raw_scale": 100, "source_record": "qwen-3-8-max-xhigh|terminus-2|pass-1", "source_model_id": "qwen-3-8-max-xhigh", "n_samples": 799, "reported_error": 6.4 }, { "id": "8f31cd68fc57b74d9743", "model": "DeepSeek-V4.1-Flash", "organization": "DeepSeek", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.519, "raw_score": 51.9, "raw_scale": 100, "source_record": "deepseek-v4-1-flash|terminus-2|pass-1", "source_model_id": "deepseek-v4-1-flash", "n_samples": 800, "reported_error": 6.3 }, { "id": "e120a16e16af45b71972", "model": "Grok 4.5", "organization": "xAI", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.5117, "raw_score": 51.17, "raw_scale": 100, "source_record": "grok-4-5|inspect|pass-1", "source_model_id": "grok-4-5", "n_samples": 800, "reported_error": 6 }, { "id": "b199b95fdfcebd4b75e5", "model": "Qwen 3.8 27B", "organization": "Alibaba", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.507, "raw_score": 50.7, "raw_scale": 100, "source_record": "qwen-3-8-27b|terminus-2|pass-1", "source_model_id": "qwen-3-8-27b", "n_samples": 800, "reported_error": 6.2 }, { "id": "b72cbe43d5081d3e103b", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.49200000000000005, "raw_score": 49.2, "raw_scale": 100, "source_record": "gemini-4-argon|terminus-2|pass-1", "source_model_id": "gemini-4-argon", "n_samples": 800, "reported_error": 6.6 }, { "id": "9f440cf7272bae90db02", "model": "DeepSeek-V4.1-Flash-1017", "organization": "DeepSeek", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.491, "raw_score": 49.1, "raw_scale": 100, "source_record": "deepseek-v4-1-flash-1017|terminus-2|pass-1", "source_model_id": "deepseek-v4-1-flash-1017", "n_samples": 800, "reported_error": 6.3 }, { "id": "1a0c5c2cc322fb612a8a", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.486, "raw_score": 48.6, "raw_scale": 100, "source_record": "gpt-6-astra-max|terminus-2|pass-1", "source_model_id": "gpt-6-astra-max", "n_samples": 1600, "reported_error": 6.3 }, { "id": "8276d881d48f1d4f2d9d", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.48, "raw_score": 48.0, "raw_scale": 100, "source_record": "gpt-5-6-sol-max|terminus-2|pass-1", "source_model_id": "gpt-5-6-sol-max", "n_samples": 800, "reported_error": 6.4 }, { "id": "d045f36c36a3d4f4a4c3", "model": "Kimi K3", "organization": "Kimi", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.48, "raw_score": 48.0, "raw_scale": 100, "source_record": "kimi-k3|terminus-2|pass-1", "source_model_id": "kimi-k3", "n_samples": 800, "reported_error": 6.2 }, { "id": "929bd0f400d1fd83fd87", "model": "Opus 4.7", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.47600000000000003, "raw_score": 47.6, "raw_scale": 100, "source_record": "claude-opus-4-7|terminus-2|pass-1", "source_model_id": "claude-opus-4-7", "n_samples": 800, "reported_error": 6.5 }, { "id": "e781c22e63de4c9cd169", "model": "Opus 4.8", "organization": "Anthropic", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.473, "raw_score": 47.3, "raw_scale": 100, "source_record": "claude-opus-4-8-high|inspect|pass-1", "source_model_id": "claude-opus-4-8-high", "n_samples": null, "reported_error": 6 }, { "id": "5e42699fd6b8e1bfcedc", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.469, "raw_score": 46.9, "raw_scale": 100, "source_record": "gpt-6-1-sol|terminus-2|pass-1", "source_model_id": "gpt-6-1-sol", "n_samples": 800, "reported_error": 6.4 }, { "id": "7fef8e83823fc2d941fd", "model": "DeepSeek-V4-Flash", "organization": "DeepSeek", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.467, "raw_score": 46.7, "raw_scale": 100, "source_record": "deepseek-v4-flash|terminus-2|pass-1", "source_model_id": "deepseek-v4-flash", "n_samples": 1600, "reported_error": 6 }, { "id": "22a83008eab95a45b06b", "model": "Sonnet 5", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.46399999999999997, "raw_score": 46.4, "raw_scale": 100, "source_record": "claude-sonnet-5-max|terminus-2|pass-1", "source_model_id": "claude-sonnet-5-max", "n_samples": 800, "reported_error": 6.4 }, { "id": "5559c3de9f5070be2660", "model": "GLM-5.3-Flash", "organization": "Zhipu", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.461, "raw_score": 46.1, "raw_scale": 100, "source_record": "glm-5-3-flash|terminus-2|pass-1", "source_model_id": "glm-5-3-flash", "n_samples": 800, "reported_error": 6.1 }, { "id": "d95584f6d01772fda2a9", "model": "DeepSeek-V4-Pro-0813", "organization": "DeepSeek", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.45899999999999996, "raw_score": 45.9, "raw_scale": 100, "source_record": "deepseek-v4-pro-08-13|terminus-2|pass-1", "source_model_id": "deepseek-v4-pro-08-13", "n_samples": 1570, "reported_error": 6.1 }, { "id": "85d71cff22d08e574691", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.45799999999999996, "raw_score": 45.8, "raw_scale": 100, "source_record": "gpt-5-6-sol-xhigh|terminus-2|pass-1", "source_model_id": "gpt-5-6-sol-xhigh", "n_samples": 800, "reported_error": 6.2 }, { "id": "3b932fb816b22da159c4", "model": "GLM-5.3", "organization": "Zhipu", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.456, "raw_score": 45.6, "raw_scale": 100, "source_record": "glm-5-3|terminus-2|pass-1", "source_model_id": "glm-5-3", "n_samples": 800, "reported_error": 5.9 }, { "id": "0047affae9656f2b465f", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.45, "raw_score": 45.0, "raw_scale": 100, "source_record": "gpt-6-sol|terminus-2|pass-1", "source_model_id": "gpt-6-sol", "n_samples": 800, "reported_error": 6.3 }, { "id": "de1a3d7ce3ee31e6ef9f", "model": "Kimi K3", "organization": "Kimi", "configuration": "Agent: Inspect; reasoning=max; pass-1", "score": 0.44670000000000004, "raw_score": 44.67, "raw_scale": 100, "source_record": "kimi-k3|inspect|pass-1", "source_model_id": "kimi-k3", "n_samples": 800, "reported_error": 6 }, { "id": "979ee83b162db42fab1c", "model": "Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=medium; pass-1", "score": 0.44, "raw_score": 44.0, "raw_scale": 100, "source_record": "claude-sonnet-5-5-medium|terminus-2|pass-1", "source_model_id": "claude-sonnet-5-5-medium", "n_samples": 800, "reported_error": 6.1 }, { "id": "9df1a61df3ab73d7c3f1", "model": "Opus 4.8", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.439, "raw_score": 43.9, "raw_scale": 100, "source_record": "claude-opus-4-8|terminus-2|pass-1", "source_model_id": "claude-opus-4-8", "n_samples": 800, "reported_error": 6.1 }, { "id": "21787b0b0abb97b987cc", "model": "Sonnet 5", "organization": "Anthropic", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.4365, "raw_score": 43.65, "raw_scale": 100, "source_record": "claude-sonnet-5|inspect|pass-1", "source_model_id": "claude-sonnet-5", "n_samples": null, "reported_error": 6 }, { "id": "63755d10c0568ef36ac4", "model": "Mercury 3", "organization": "Inception", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.431, "raw_score": 43.1, "raw_scale": 100, "source_record": "inception/mercury-3|terminus-2|pass-1", "source_model_id": "inception/mercury-3", "n_samples": 800, "reported_error": 6 }, { "id": "6a39ca5910e5d6798d87", "model": "Gemini 3.7 Flash", "organization": "Google", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.42, "raw_score": 42.0, "raw_scale": 100, "source_record": "gemini-3.7-flash|terminus-2|pass-1", "source_model_id": "gemini-3.7-flash", "n_samples": 800, "reported_error": 6.4 }, { "id": "c7f552918322de9b1d4c", "model": "Opus 4.6", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.418, "raw_score": 41.8, "raw_scale": 100, "source_record": "claude-opus-4-6-max|terminus-2|pass-1", "source_model_id": "claude-opus-4-6-max", "n_samples": 795, "reported_error": 6.2 }, { "id": "48296867276b1e57e680", "model": "Beam", "organization": "Reflection", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.41600000000000004, "raw_score": 41.6, "raw_scale": 100, "source_record": "beam-1-0-max|terminus-2|pass-1", "source_model_id": "beam-1-0-max", "n_samples": 800, "reported_error": 5.9 }, { "id": "8e7e870ebfc90eb1b52c", "model": "GPT-5.3 Codex", "organization": "OpenAI", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.415, "raw_score": 41.5, "raw_scale": 100, "source_record": "gpt-5.3-codex-high|inspect|pass-1", "source_model_id": "gpt-5.3-codex-high", "n_samples": null, "reported_error": 6.25 }, { "id": "3913915f4092bc158758", "model": "Opus 4.7", "organization": "Anthropic", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.41340000000000005, "raw_score": 41.34, "raw_scale": 100, "source_record": "claude-opus-4-7-high|inspect|pass-1", "source_model_id": "claude-opus-4-7-high", "n_samples": null, "reported_error": 6.33 }, { "id": "dc83f20cebc330357fff", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: Inspect; reasoning=xhigh; pass-1", "score": 0.40840000000000004, "raw_score": 40.84, "raw_scale": 100, "source_record": "gpt-5.5-xhigh|inspect|pass-1", "source_model_id": "gpt-5.5-xhigh", "n_samples": 1600, "reported_error": 6.5 }, { "id": "2556cf4e4a468e8f16a9", "model": "Muse Spark 1.2", "organization": "Meta", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "muse-spark-1-2|terminus-2|pass-1", "source_model_id": "muse-spark-1-2", "n_samples": 800, "reported_error": 6.2 }, { "id": "9ec13606cc2da3c1b332", "model": "Kimi K2.7 Code", "organization": "Kimi", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.40299999999999997, "raw_score": 40.3, "raw_scale": 100, "source_record": "kimi-k2-7-code-high|terminus-2|pass-1", "source_model_id": "kimi-k2-7-code-high", "n_samples": 800, "reported_error": 5.9 }, { "id": "eb7ed2717717d78563bd", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Agent: Inspect; reasoning=xhigh; pass-1", "score": 0.3967, "raw_score": 39.67, "raw_scale": 100, "source_record": "gpt-5-6-sol-xhigh|inspect|pass-1", "source_model_id": "gpt-5-6-sol-xhigh", "n_samples": 800, "reported_error": 5.5 }, { "id": "c3bed4744ed78c76f77b", "model": "Gemini 3.6 Flash", "organization": "Google", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.3967, "raw_score": 39.67, "raw_scale": 100, "source_record": "gemini-3.6-flash|inspect|pass-1", "source_model_id": "gemini-3.6-flash", "n_samples": 800, "reported_error": 6 }, { "id": "73df7d826c5a97a10038", "model": "Gemini 3.6 Flash", "organization": "Google", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.39399999999999996, "raw_score": 39.4, "raw_scale": 100, "source_record": "gemini-3.6-flash|terminus-2|pass-1", "source_model_id": "gemini-3.6-flash", "n_samples": 800, "reported_error": 6.2 }, { "id": "d39aa0b935b2ca100b88", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.391, "raw_score": 39.1, "raw_scale": 100, "source_record": "muse-spark-1-3-max|terminus-2|pass-1", "source_model_id": "muse-spark-1-3-max", "n_samples": 1592, "reported_error": 6.4 }, { "id": "f1056b79b92ca6621067", "model": "Muse Spark 1.1", "organization": "Meta", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.389, "raw_score": 38.9, "raw_scale": 100, "source_record": "muse-spark-1-1|terminus-2|pass-1", "source_model_id": "muse-spark-1-1", "n_samples": 656, "reported_error": 6.1 }, { "id": "b978b472badd2860cc55", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.38799999999999996, "raw_score": 38.8, "raw_scale": 100, "source_record": "gpt-6-luna|terminus-2|pass-1", "source_model_id": "gpt-6-luna", "n_samples": 800, "reported_error": 6.1 }, { "id": "b49580e8e93cf64f068e", "model": "GLM-5.2", "organization": "Zhipu", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.374, "raw_score": 37.4, "raw_scale": 100, "source_record": "glm-5-2|terminus-2|pass-1", "source_model_id": "glm-5-2", "n_samples": 800, "reported_error": 6.1 }, { "id": "813425925d11aa7caac0", "model": "GLM-5.2", "organization": "Zhipu", "configuration": "Agent: Inspect; pass-1", "score": 0.3732, "raw_score": 37.32, "raw_scale": 100, "source_record": "zai-org/GLM-5.2-FP8|inspect|pass-1", "source_model_id": "zai-org/GLM-5.2-FP8", "n_samples": null, "reported_error": 5.5 }, { "id": "2784008198028c7363f3", "model": "Mistral Large 4", "organization": "Mistral", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.37, "raw_score": 37.0, "raw_scale": 100, "source_record": "mistral-large-4|terminus-2|pass-1", "source_model_id": "mistral-large-4", "n_samples": 800, "reported_error": 5.8 }, { "id": "c6c0824ee90793963f4d", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.366, "raw_score": 36.6, "raw_scale": 100, "source_record": "gpt-5.5-xhigh|terminus-2|pass-1", "source_model_id": "gpt-5.5-xhigh", "n_samples": 1600, "reported_error": 5.8 }, { "id": "0c09f5f094f4537fd3de", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.364, "raw_score": 36.4, "raw_scale": 100, "source_record": "gemini-3.8-flash|terminus-2|pass-1", "source_model_id": "gemini-3.8-flash", "n_samples": 1595, "reported_error": 5.9 }, { "id": "e1b07023d0505298a138", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.364, "raw_score": 36.4, "raw_scale": 100, "source_record": "muse-spark-1-3|terminus-2|pass-1", "source_model_id": "muse-spark-1-3", "n_samples": 796, "reported_error": 6.4 }, { "id": "66ac9a7f4cfb365f1795", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.361, "raw_score": 36.1, "raw_scale": 100, "source_record": "gemini-3.5-flash|terminus-2|pass-1", "source_model_id": "gemini-3.5-flash", "n_samples": 800, "reported_error": 6 }, { "id": "a26308cae2345468b209", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.34600000000000003, "raw_score": 34.6, "raw_scale": 100, "source_record": "minimax-m3|terminus-2|pass-1", "source_model_id": "minimax-m3", "n_samples": 1600, "reported_error": 5.6 }, { "id": "052d829db36a47802b2d", "model": "GLM-5.1", "organization": "Zhipu", "configuration": "Agent: Terminus-2; pass-1", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "glm-5-1|terminus-2|pass-1", "source_model_id": "glm-5-1", "n_samples": 800, "reported_error": 5.7 }, { "id": "03fc65522cd2efb49505", "model": "Sonnet 4.6", "organization": "Anthropic", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "claude-sonnet-4-6-high|terminus-2|pass-1", "source_model_id": "claude-sonnet-4-6-high", "n_samples": 800, "reported_error": 6.1 }, { "id": "281eea77f89e4d771e3f", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=xhigh; pass-1", "score": 0.34299999999999997, "raw_score": 34.3, "raw_scale": 100, "source_record": "responses/gpt-5.4|terminus-2|pass-1", "source_model_id": "responses/gpt-5.4", "n_samples": 800, "reported_error": 5.5 }, { "id": "6d0972b82541054d79b0", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.33899999999999997, "raw_score": 33.9, "raw_scale": 100, "source_record": "gemini-3.1-pro-preview-high|terminus-2|pass-1", "source_model_id": "gemini-3.1-pro-preview-high", "n_samples": 800, "reported_error": 5.8 }, { "id": "5a524a91484eb27bc464", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.33799999999999997, "raw_score": 33.8, "raw_scale": 100, "source_record": "deepseek-v4-pro|terminus-2|pass-1", "source_model_id": "deepseek-v4-pro", "n_samples": 800, "reported_error": 5.7 }, { "id": "350336f8c9beff5c6f80", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.325, "raw_score": 32.5, "raw_scale": 100, "source_record": "gemini-3.5-flash|inspect|pass-1", "source_model_id": "gemini-3.5-flash", "n_samples": 800, "reported_error": 5.8 }, { "id": "362edd3d8fc1376deaea", "model": "Kimi K2.7", "organization": "Kimi", "configuration": "Agent: Inspect; pass-1", "score": 0.3248, "raw_score": 32.48, "raw_scale": 100, "source_record": "moonshotai/Kimi-K2.7|inspect|pass-1", "source_model_id": "moonshotai/Kimi-K2.7", "n_samples": null, "reported_error": 6 }, { "id": "d4b84b83360dce36fbca", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.321, "raw_score": 32.1, "raw_scale": 100, "source_record": "gpt-5.6-luna-max|terminus-2|pass-1", "source_model_id": "gpt-5.6-luna-max", "n_samples": 800, "reported_error": 5.6 }, { "id": "c176c39261e4f735e2eb", "model": "Cognition SWE-1.6 Preview", "organization": "", "configuration": "Agent: Inspect; pass-1", "score": 0.317, "raw_score": 31.7, "raw_scale": 100, "source_record": "cognition-swe-1-6|inspect|pass-1", "source_model_id": "cognition-swe-1-6", "n_samples": null, "reported_error": 5.75 }, { "id": "3559c01a292ac93e660f", "model": "Gemini 3.5 Flash Lite", "organization": "Google", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.312, "raw_score": 31.2, "raw_scale": 100, "source_record": "gemini-3-5-flash-lite|terminus-2|pass-1", "source_model_id": "gemini-3-5-flash-lite", "n_samples": 800, "reported_error": 5.6 }, { "id": "b9d8b04d051c9e8fcb68", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=max; pass-1", "score": 0.311, "raw_score": 31.1, "raw_scale": 100, "source_record": "gpt-5.6-terra-max|terminus-2|pass-1", "source_model_id": "gpt-5.6-terra-max", "n_samples": 800, "reported_error": 5.6 }, { "id": "ba7f97c1e8a299989143", "model": "Sonnet 4.5", "organization": "Anthropic", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.31, "raw_score": 31.0, "raw_scale": 100, "source_record": "sonnet-4.5-high|inspect|pass-1", "source_model_id": "sonnet-4.5-high", "n_samples": null, "reported_error": 6 }, { "id": "307b250c31139f01a7a1", "model": "GPT-5.2 Codex", "organization": "OpenAI", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.308, "raw_score": 30.8, "raw_scale": 100, "source_record": "gpt-5.2-codex-high|inspect|pass-1", "source_model_id": "gpt-5.2-codex-high", "n_samples": null, "reported_error": 5.84 }, { "id": "59e2540f0b1a9772aca8", "model": "Kimi K2.5", "organization": "Kimi", "configuration": "Agent: Inspect; pass-1", "score": 0.303, "raw_score": 30.3, "raw_scale": 100, "source_record": "moonshotai/Kimi-K2.5|inspect|pass-1", "source_model_id": "moonshotai/Kimi-K2.5", "n_samples": null, "reported_error": 5.83 }, { "id": "76f90b9632c4b59dd504", "model": "GPT-5.1 Codex", "organization": "OpenAI", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.29, "raw_score": 29.0, "raw_scale": 100, "source_record": "gpt-5.1-codex-high|inspect|pass-1", "source_model_id": "gpt-5.1-codex-high", "n_samples": null, "reported_error": 5.59 }, { "id": "2b881043a16eec8ae828", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.28800000000000003, "raw_score": 28.8, "raw_scale": 100, "source_record": "gemini-3.1-pro-preview-high|inspect|pass-1", "source_model_id": "gemini-3.1-pro-preview-high", "n_samples": 800, "reported_error": 5.42 }, { "id": "bbb99e2aabd7cde4fcc8", "model": "Qwen3.5", "organization": "Alibaba", "configuration": "Agent: Terminus-2; pass-1", "score": 0.27899999999999997, "raw_score": 27.9, "raw_scale": 100, "source_record": "Qwen3.5-397B-A17B|terminus-2|pass-1", "source_model_id": "Qwen3.5-397B-A17B", "n_samples": 2399, "reported_error": 5.1 }, { "id": "74a87929e615cd810678", "model": "Gemini 3 Pro", "organization": "Google", "configuration": "Agent: Inspect; reasoning=high; pass-1", "score": 0.268, "raw_score": 26.8, "raw_scale": 100, "source_record": "gemini-3-pro-preview-high|inspect|pass-1", "source_model_id": "gemini-3-pro-preview-high", "n_samples": null, "reported_error": 5.5 }, { "id": "357bcd73f95ed163715c", "model": "Inkling", "organization": "Thinking Machines", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.261, "raw_score": 26.1, "raw_scale": 100, "source_record": "inkling|terminus-2|pass-1", "source_model_id": "inkling", "n_samples": 800, "reported_error": 5.2 }, { "id": "4007bd82d26073ff4a6a", "model": "DeepSeek-V3.2", "organization": "DeepSeek", "configuration": "Agent: Terminus-2; pass-1", "score": 0.256, "raw_score": 25.6, "raw_scale": 100, "source_record": "deepseek-ai/DeepSeek-V3.2|terminus-2|pass-1", "source_model_id": "deepseek-ai/DeepSeek-V3.2", "n_samples": 800, "reported_error": 5.1 }, { "id": "3dd104edaa98abc6aed2", "model": "Nemotron 3.5 Super VL", "organization": "NVIDIA", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "nemotron-3-5-super-vl|terminus-2|pass-1", "source_model_id": "nemotron-3-5-super-vl", "n_samples": 800, "reported_error": 4.9 }, { "id": "778890e2d0221a3511f2", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.215, "raw_score": 21.5, "raw_scale": 100, "source_record": "minimax-m2-7|terminus-2|pass-1", "source_model_id": "minimax-m2-7", "n_samples": 800, "reported_error": 4.6 }, { "id": "9abd9e1b13df16122194", "model": "Grok 4", "organization": "xAI", "configuration": "Agent: Inspect; pass-1", "score": 0.21, "raw_score": 21.0, "raw_scale": 100, "source_record": "grok-4|inspect|pass-1", "source_model_id": "grok-4", "n_samples": null, "reported_error": 5 }, { "id": "eb1b8772be9019d31a50", "model": "Kimi K2", "organization": "Kimi", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.174, "raw_score": 17.4, "raw_scale": 100, "source_record": "kimi-k2-thinking-high|terminus-2|pass-1", "source_model_id": "kimi-k2-thinking-high", "n_samples": 800, "reported_error": 4.2 }, { "id": "d4985b65fcb3eb79a1e4", "model": "DeepSeek-V3.2", "organization": "DeepSeek", "configuration": "Agent: Inspect; pass-1", "score": 0.158, "raw_score": 15.8, "raw_scale": 100, "source_record": "deepseek-ai/DeepSeek-V3.2|inspect|pass-1", "source_model_id": "deepseek-ai/DeepSeek-V3.2", "n_samples": 800, "reported_error": 4.33 }, { "id": "1b351d36ba800a4662b1", "model": "Nemotron 3 Ultra", "organization": "NVIDIA", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.152, "raw_score": 15.2, "raw_scale": 100, "source_record": "nemotron-3-ultra-nvfp4|terminus-2|pass-1", "source_model_id": "nemotron-3-ultra-nvfp4", "n_samples": 1579, "reported_error": 3.6 }, { "id": "ec09e5b50f1ec8b6778a", "model": "Kimi K2", "organization": "Kimi", "configuration": "Agent: Inspect; pass-1", "score": 0.11199999999999999, "raw_score": 11.2, "raw_scale": 100, "source_record": "moonshotai/Kimi-K2-Thinking|inspect|pass-1", "source_model_id": "moonshotai/Kimi-K2-Thinking", "n_samples": null, "reported_error": 3.58 }, { "id": "a79476aef597765d19c2", "model": "GPT-OSS-120B", "organization": "OpenAI", "configuration": "Agent: Terminus-2; reasoning=high; pass-1", "score": 0.033, "raw_score": 3.3, "raw_scale": 100, "source_record": "openai/gpt-oss-120b|terminus-2|pass-1", "source_model_id": "openai/gpt-oss-120b", "n_samples": 800, "reported_error": 1.6 } ] }, { "id": "frontierswe-v2", "name": "FrontierSWE v2", "group": "编程", "unit": "score", "metric": "Mean@5 score", "protocol": { "en": "34 tasks; proximus; five trials and a 20-hour budget per task. Absolute mean@5 score (0–100), including partial-credit performance tasks; not best@5 or relative-to-frontier normalization.", "zh": "34 道任务；proximus；每题五次试验、每次 20 小时。使用 0–100 的绝对 mean@5 得分，包含性能任务的部分得分；不使用 best@5 或相对最优模型归一化。" }, "source": { "name": "Proximal", "benchmark_creator": "Proximal", "url": "https://www.frontierswe.com/", "data_url": "https://www.frontierswe.com/", "sha256": "9e1f1568276e81c8d059fed43f0857592d50f348e3b7b38d24925c1e1247d957", "retrieved_at": "2026-11-02T03:34:26.698010+00:00", "selector": "Next.js entries.abs.mean[].overall; all models, not best-per-provider UI subset", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "8ae240bfabca74edf28b", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 65.50757455882352, "raw_score": 65.50757455882352, "raw_scale": 1, "source_record": "GPT-6 Astra|proximus", "report_url": "https://www.frontierswe.com/traces?model=gpt-6-astra" }, { "id": "195119365cb769904962", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 62.33098823529411, "raw_score": 62.33098823529411, "raw_scale": 1, "source_record": "Claude Opus 5.5|proximus", "report_url": "https://www.frontierswe.com/traces?model=claude-opus-5.5" }, { "id": "58962446c1f62ba391c2", "model": "Claude Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 61.889299411764696, "raw_score": 61.889299411764696, "raw_scale": 1, "source_record": "Claude Sonnet 5.5|proximus", "report_url": "https://www.frontierswe.com/traces?model=claude-sonnet-5.5" }, { "id": "0dc01b9d5f903895340f", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 61.19949235294118, "raw_score": 61.19949235294118, "raw_scale": 1, "source_record": "Gemini 4 Carbon|proximus", "report_url": "https://www.frontierswe.com/traces?model=gemini-4-carbon" }, { "id": "e71987e97ead1dcd11b6", "model": "Claude Fable 5.1", "organization": "Anthropic", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 56.29078999999999, "raw_score": 56.29078999999999, "raw_scale": 1, "source_record": "Claude Fable 5.1|proximus", "report_url": "https://www.frontierswe.com/traces?model=claude-fable-5.1" }, { "id": "825dbd3f90bf59145a29", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 54.96118588235292, "raw_score": 54.96118588235292, "raw_scale": 1, "source_record": "Gemini 4 Argon|proximus", "report_url": "https://www.frontierswe.com/traces?model=gemini-4-argon" }, { "id": "07efd79da877b1b2426b", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 52.01176764705882, "raw_score": 52.01176764705882, "raw_scale": 1, "source_record": "Claude Opus 5|proximus", "report_url": "https://www.frontierswe.com/traces?model=claude-opus-5" }, { "id": "30d419f443a884ecdf38", "model": "Claude Fable 5", "organization": "Anthropic", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 46.96449352941177, "raw_score": 46.96449352941177, "raw_scale": 1, "source_record": "Claude Fable 5|proximus", "report_url": "https://www.frontierswe.com/traces?model=claude-fable-5" }, { "id": "52e48ff34d04737a0f16", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 43.76868294117648, "raw_score": 43.76868294117648, "raw_scale": 1, "source_record": "Claude Haiku 5.5|proximus", "report_url": "https://www.frontierswe.com/traces?model=claude-haiku-5.5" }, { "id": "2b59a8a9901fef53e0cb", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 32.20246705882353, "raw_score": 32.20246705882353, "raw_scale": 1, "source_record": "GPT-5.6 Sol|proximus", "report_url": "https://www.frontierswe.com/traces?model=gpt-5.6-sol" }, { "id": "cbc6d48060197bb3d3e1", "model": "GLM-5.3", "organization": "Z AI", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 30.181864117647052, "raw_score": 30.181864117647052, "raw_scale": 1, "source_record": "GLM-5.3|proximus", "report_url": "https://www.frontierswe.com/traces?model=glm-5.3" }, { "id": "8f4ca7ea4d57a5529dd4", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 29.50476117647059, "raw_score": 29.50476117647059, "raw_scale": 1, "source_record": "Grok 4.7|proximus", "report_url": "https://www.frontierswe.com/traces?model=grok-4.7" }, { "id": "a7fcb46be52fd99ab2be", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 25.865032058823534, "raw_score": 25.865032058823534, "raw_scale": 1, "source_record": "Kimi K3|proximus", "report_url": "https://www.frontierswe.com/traces?model=kimi-k3" }, { "id": "6c0ee43f0fc4287bbe62", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 25.80214823529412, "raw_score": 25.80214823529412, "raw_scale": 1, "source_record": "Muse Spark 1.3|proximus", "report_url": "https://www.frontierswe.com/traces?model=muse-spark-1.3" }, { "id": "64bcf39c5e841b495b73", "model": "Grok 4.6", "organization": "xAI", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 25.286195882352946, "raw_score": 25.286195882352946, "raw_scale": 1, "source_record": "Grok 4.6|proximus", "report_url": "https://www.frontierswe.com/traces?model=grok-4.6" }, { "id": "1a88eb2bac5a93e7013f", "model": "Gemini 3.7 Flash", "organization": "Google", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 20.260590588235296, "raw_score": 20.260590588235296, "raw_scale": 1, "source_record": "Gemini 3.7 Flash|proximus", "report_url": "https://www.frontierswe.com/traces?model=gemini-3.7-flash" }, { "id": "6b501431f5af362d5d5e", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 19.626902941176464, "raw_score": 19.626902941176464, "raw_scale": 1, "source_record": "Gemini 3.8 Flash|proximus", "report_url": "https://www.frontierswe.com/traces?model=gemini-3.8-flash" }, { "id": "01d77b3a87407971cb53", "model": "GLM-5.3 Flash", "organization": "Z AI", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 18.125205882352944, "raw_score": 18.125205882352944, "raw_scale": 1, "source_record": "GLM-5.3 Flash|proximus", "report_url": "https://www.frontierswe.com/traces?model=glm-5.3-flash" }, { "id": "66ad005b2f297e949a72", "model": "Qwen3.8-Max-0902", "organization": "Alibaba", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 17.809649411764706, "raw_score": 17.809649411764706, "raw_scale": 1, "source_record": "Qwen3.8-Max-0902|proximus", "report_url": "https://www.frontierswe.com/traces?model=qwen3.8-max-0902" }, { "id": "83655bfd0b09eca3f9fa", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 15.809350588235294, "raw_score": 15.809350588235294, "raw_scale": 1, "source_record": "Qwen3.8-Max|proximus", "report_url": "https://www.frontierswe.com/traces?model=qwen3.8-max" }, { "id": "f7c0da9042da12868a34", "model": "DeepSeek V4 Flash Vision Exp", "organization": "DeepSeek", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 14.831252647058824, "raw_score": 14.831252647058824, "raw_scale": 1, "source_record": "DeepSeek V4 Flash Vision Exp|proximus", "report_url": "https://www.frontierswe.com/traces?model=deepseek-v4-flash-vision-exp" }, { "id": "00d518245ec24c800195", "model": "Muse Spark 1.2", "organization": "Meta", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 11.970256470588238, "raw_score": 11.970256470588238, "raw_scale": 1, "source_record": "Muse Spark 1.2|proximus", "report_url": "https://www.frontierswe.com/traces?model=muse-spark-1.2" }, { "id": "e71fb6d7132db5b1e3d4", "model": "Inkling", "organization": "Thinking Machines Lab", "configuration": "Agent: proximus; mean@5; 20 hours per task", "score": 4.118886470588236, "raw_score": 4.118886470588236, "raw_scale": 1, "source_record": "Inkling|proximus", "report_url": "https://www.frontierswe.com/traces?model=inkling" } ], "score_max": 100 }, { "id": "turing-sdlc-all-core", "name": "SDLCbench (All-core pass)", "group": "编程", "unit": "%", "metric": "All-core pass rate", "protocol": { "en": "Private enterprise software-development tasks; all core checks must pass. Harnesses are retained separately; soft-gated rewards are not used as pass rates.", "zh": "私有企业软件开发任务；所有核心检查通过才算成功。分别保留 Agent；不将软门控奖励当作通过率。" }, "source": { "name": "Turing", "benchmark_creator": "Turing", "url": "https://bench.turing.com/sdlc/", "data_url": "https://bench.turing.com/sdlc/dataV3.json", "sha256": "f5223bf113d7b4612375ac2ac20f0d097486b7fedea5cb622020e5969a7583ed", "retrieved_at": "2026-11-02T03:36:56.872929+00:00", "selector": "leaderboard[].all_core_pass_rate", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "22f9632f4ef195b690ea", "model": "claude-fable-5", "organization": "Anthropic", "configuration": "Agent: Claude Code", "score": 0.4, "raw_score": 40.0, "raw_scale": 100, "source_record": "claude-code/claude-fable-5" }, { "id": "4ba06fa45f2129d06765", "model": "gpt-5.5", "organization": "OpenAI", "configuration": "Agent: Codex", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "codex/gpt-5.5" }, { "id": "332d8175d5342c481c88", "model": "claude-opus-4-8", "organization": "Anthropic", "configuration": "Agent: OpenCode", "score": 0.2, "raw_score": 20.0, "raw_scale": 100, "source_record": "opencode/claude-opus-4-8" }, { "id": "1adcc75e8522ae9472dd", "model": "gpt-5.5", "organization": "OpenAI", "configuration": "Agent: OpenCode", "score": 0.2, "raw_score": 20.0, "raw_scale": 100, "source_record": "opencode/gpt-5.5" }, { "id": "238dcd2ee63e14f3f30a", "model": "gemini-3.1-pro-preview", "organization": "Google", "configuration": "Agent: OpenCode", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "opencode/gemini-3.1-pro-preview" }, { "id": "ec0fb328209a39c9011f", "model": "grok-4.20-0309-reasoning", "organization": "xAI", "configuration": "Agent: OpenCode", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "opencode/grok-4.20-0309-reasoning" } ] }, { "id": "turing-companybench", "name": "CompanyBench", "group": "专业工作", "unit": "%", "metric": "All-core pass rate", "protocol": { "en": "64 tasks, ten independent trials per model; a trial passes only when every check passes. Published June 2026 evaluation cohort.", "zh": "64 道任务，每模型十次独立试验；每项检查均通过时才算成功。官方 2026 年 6 月评测批次。" }, "source": { "name": "Turing", "benchmark_creator": "Turing", "url": "https://bench.turing.com/companybench/leaderboard/", "data_url": "https://bench.turing.com/companybench/leaderboard/", "sha256": "abc43b6b0f3011d2e17f9bd024cb3773dbc73c179fb6da7b2f33735d7a55bcd5", "retrieved_at": "2026-11-02T03:36:56.219636+00:00", "selector": "[data-board-rank][data-model].data-sort-score; full task pass fraction", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "961895edd3c8fdc42a61", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Published task-level pass rate", "score": 0.4438, "raw_score": 0.4438, "raw_scale": 1, "source_record": "GPT-5.5|Published task-level pass rate" }, { "id": "b59f0b04f90ff31f5587", "model": "Claude Opus 4.8", "organization": "Anthropic", "configuration": "Published task-level pass rate", "score": 0.3594, "raw_score": 0.3594, "raw_scale": 1, "source_record": "Claude Opus 4.8|Published task-level pass rate" }, { "id": "89675b1d12ce3bd4731a", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "Published task-level pass rate", "score": 0.1828, "raw_score": 0.1828, "raw_scale": 1, "source_record": "Gemini 3.1 Pro|Published task-level pass rate" }, { "id": "950939b3bad6592d20d5", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Published task-level pass rate", "score": 0.0766, "raw_score": 0.0766, "raw_scale": 1, "source_record": "Gemini 3.5 Flash|Published task-level pass rate" } ] }, { "id": "turing-scicode-plus-plus", "name": "SciCode++", "group": "数学与科学", "unit": "%", "metric": "Main problem pass@8 (publisher label)", "protocol": { "en": "The shared 3,451-problem grid, eight trials per model; official Main problem pass@8 column. Both published models use high reasoning. Distinct from SciCode.", "zh": "共同完成的 3,451 道题，每模型八次试验；沿用官方 Main problem pass@8 列。两模型均使用 high 推理。与 SciCode 分开。" }, "source": { "name": "Turing", "benchmark_creator": "Turing", "url": "https://bench.turing.com/scicode/leaderboard/", "data_url": "https://bench.turing.com/scicode/leaderboard/", "sha256": "4c980082874adb45568d3d2a99069050e9ea093cb37ccaa41220c132c13b46e1", "retrieved_at": "2026-11-02T03:38:01.237318+00:00", "selector": "[data-board-rank][data-model].data-sort-score; official Main problem pass@8 column", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "4ec46b4ac5aa190a7af2", "model": "GPT-5.6 (sol)", "organization": "OpenAI", "configuration": "reasoning=high; eight trials; shared 3451-problem grid", "score": 0.1771, "raw_score": 0.1771, "raw_scale": 1, "source_record": "GPT-5.6 (sol)|Published task-level pass rate" }, { "id": "2c158496d10061d2aede", "model": "Gemini 3.1 Pro Preview", "organization": "Google", "configuration": "reasoning=high; eight trials; shared 3451-problem grid", "score": 0.1464, "raw_score": 0.1464, "raw_scale": 1, "source_record": "Gemini 3.1 Pro Preview|Published task-level pass rate" } ] }, { "id": "skillsbench-with-skills", "name": "SkillsBench (With Skills)", "group": "Agent 与工具", "unit": "%", "metric": "Task pass rate", "protocol": { "en": "Official paired leaderboard, with skills; model and agent pairs remain distinct. Uses pass rate, not normalized gain.", "zh": "官方配对评测榜单；使用 Skills；分别记录模型与 Agent。使用通过率，不使用归一化增益。" }, "source": { "name": "BenchFlow", "benchmark_creator": "BenchFlow", "url": "https://www.skillsbench.ai/", "data_url": "https://www.skillsbench.ai/", "sha256": "956ab5a9313b10e5e10fb23ff5ac1c172dad5a1acb8da34af454508444f2e82b", "retrieved_at": "2026-11-02T03:36:56.496569+00:00", "selector": "paired leaderboard table: With Skills", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "0e14892c15944fdfb09a", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: OpenHands; skills=true", "score": 0.6729999999999999, "raw_score": 67.3, "raw_scale": 100, "source_record": "GPT-5.5|Agent: OpenHands; skills=true" }, { "id": "dc65bca0865e54f840ca", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: Codex; skills=true", "score": 0.665, "raw_score": 66.5, "raw_scale": 100, "source_record": "GPT-5.5|Agent: Codex; skills=true" }, { "id": "65bdf4b38d07e5b8accd", "model": "Opus 4.7", "organization": "Claude", "configuration": "Agent: Claude Code; skills=true", "score": 0.612, "raw_score": 61.2, "raw_scale": 100, "source_record": "Opus 4.7|Agent: Claude Code; skills=true" }, { "id": "46ad8731f4d45c08659a", "model": "Gemini 3.1 Pro", "organization": "Gemini", "configuration": "Agent: Gemini CLI; skills=true", "score": 0.608, "raw_score": 60.8, "raw_scale": 100, "source_record": "Gemini 3.1 Pro|Agent: Gemini CLI; skills=true" }, { "id": "0c7dfc4ffc3b640e174f", "model": "GLM 5.1", "organization": "Z.ai", "configuration": "Agent: OpenHands; skills=true", "score": 0.584, "raw_score": 58.4, "raw_scale": 100, "source_record": "GLM 5.1|Agent: OpenHands; skills=true" }, { "id": "d174f02b02f6df5b8a8c", "model": "HY3", "organization": "Hunyuan", "configuration": "Agent: Claude Code; skills=true", "score": 0.5589999999999999, "raw_score": 55.9, "raw_scale": 100, "source_record": "HY3|Agent: Claude Code; skills=true" }, { "id": "2c3c41058ec1d5f30914", "model": "Gemini 3 Flash", "organization": "Gemini", "configuration": "Agent: Gemini CLI; skills=true", "score": 0.546, "raw_score": 54.6, "raw_scale": 100, "source_record": "Gemini 3 Flash|Agent: Gemini CLI; skills=true" }, { "id": "878847b09b6ea5e86343", "model": "Opus 4.8", "organization": "Claude", "configuration": "Agent: OpenHands; skills=true", "score": 0.541, "raw_score": 54.1, "raw_scale": 100, "source_record": "Opus 4.8|Agent: OpenHands; skills=true" }, { "id": "f2e569b87fd5e816378f", "model": "Kimi K2.6", "organization": "MoonshotAI", "configuration": "Agent: OpenHands; skills=true", "score": 0.54, "raw_score": 54.0, "raw_scale": 100, "source_record": "Kimi K2.6|Agent: OpenHands; skills=true" }, { "id": "9ecc4e6a45ceb251501f", "model": "Opus 4.7", "organization": "Claude", "configuration": "Agent: OpenHands; skills=true", "score": 0.531, "raw_score": 53.1, "raw_scale": 100, "source_record": "Opus 4.7|Agent: OpenHands; skills=true" }, { "id": "6f07411cedd0d1cece42", "model": "MiniMax M3", "organization": "Minimax", "configuration": "Agent: OpenHands; skills=true", "score": 0.53, "raw_score": 53.0, "raw_scale": 100, "source_record": "MiniMax M3|Agent: OpenHands; skills=true" }, { "id": "3c94c91d638906d77f9f", "model": "Gemini 3.1 Pro", "organization": "Gemini", "configuration": "Agent: OpenHands; skills=true", "score": 0.528, "raw_score": 52.8, "raw_scale": 100, "source_record": "Gemini 3.1 Pro|Agent: OpenHands; skills=true" }, { "id": "d7de6df78fd1d0bdef4e", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Agent: Codex; skills=true", "score": 0.517, "raw_score": 51.7, "raw_scale": 100, "source_record": "GPT-5.2|Agent: Codex; skills=true" }, { "id": "8a861b28ec9c72011a79", "model": "Opus 4.6", "organization": "Claude", "configuration": "Agent: Claude Code; skills=true", "score": 0.502, "raw_score": 50.2, "raw_scale": 100, "source_record": "Opus 4.6|Agent: Claude Code; skills=true" }, { "id": "2940d99125f373eaab28", "model": "DeepSeek V4 Pro", "organization": "DeepSeek", "configuration": "Agent: OpenHands; skills=true", "score": 0.501, "raw_score": 50.1, "raw_scale": 100, "source_record": "DeepSeek V4 Pro|Agent: OpenHands; skills=true" }, { "id": "f2a471e4d452d19c80b0", "model": "Opus 4.5", "organization": "Claude", "configuration": "Agent: Claude Code; skills=true", "score": 0.49, "raw_score": 49.0, "raw_scale": 100, "source_record": "Opus 4.5|Agent: Claude Code; skills=true" }, { "id": "e3401bb7398bf7e9c0e2", "model": "Gemini 3.5 Flash", "organization": "Gemini", "configuration": "Agent: OpenHands; skills=true", "score": 0.48200000000000004, "raw_score": 48.2, "raw_scale": 100, "source_record": "Gemini 3.5 Flash|Agent: OpenHands; skills=true" }, { "id": "ab72373b941b0102df1e", "model": "Sonnet 4.6", "organization": "Claude", "configuration": "Agent: OpenHands; skills=true", "score": 0.47200000000000003, "raw_score": 47.2, "raw_scale": 100, "source_record": "Sonnet 4.6|Agent: OpenHands; skills=true" }, { "id": "c8e3b80061a605892617", "model": "DeepSeek V4 Flash", "organization": "DeepSeek", "configuration": "Agent: OpenHands; skills=true", "score": 0.447, "raw_score": 44.7, "raw_scale": 100, "source_record": "DeepSeek V4 Flash|Agent: OpenHands; skills=true" }, { "id": "656e468886d0997cc86e", "model": "Grok 4.3", "organization": "Grok", "configuration": "Agent: OpenHands; skills=true", "score": 0.41700000000000004, "raw_score": 41.7, "raw_scale": 100, "source_record": "Grok 4.3|Agent: OpenHands; skills=true" }, { "id": "4f299bd57a684a130585", "model": "GPT-5.4 Mini", "organization": "OpenAI", "configuration": "Agent: OpenHands; skills=true", "score": 0.414, "raw_score": 41.4, "raw_scale": 100, "source_record": "GPT-5.4 Mini|Agent: OpenHands; skills=true" }, { "id": "62bbb0024551b4ec0e4f", "model": "Sonnet 4.5", "organization": "Claude", "configuration": "Agent: Claude Code; skills=true", "score": 0.36200000000000004, "raw_score": 36.2, "raw_scale": 100, "source_record": "Sonnet 4.5|Agent: Claude Code; skills=true" }, { "id": "c323412734ce70f309aa", "model": "MiniMax M2.7", "organization": "Minimax", "configuration": "Agent: OpenHands; skills=true", "score": 0.349, "raw_score": 34.9, "raw_scale": 100, "source_record": "MiniMax M2.7|Agent: OpenHands; skills=true" }, { "id": "6ab45f33b8d2827cd2d1", "model": "Haiku 4.5", "organization": "Claude", "configuration": "Agent: Claude Code; skills=true", "score": 0.301, "raw_score": 30.1, "raw_scale": 100, "source_record": "Haiku 4.5|Agent: Claude Code; skills=true" }, { "id": "1d75a1362188c905a8e9", "model": "Gemini 3.1 Flash Lite", "organization": "Gemini", "configuration": "Agent: OpenHands; skills=true", "score": 0.201, "raw_score": 20.1, "raw_scale": 100, "source_record": "Gemini 3.1 Flash Lite|Agent: OpenHands; skills=true" } ] }, { "id": "skillsbench-without-skills", "name": "SkillsBench (Without Skills)", "group": "Agent 与工具", "unit": "%", "metric": "Task pass rate", "protocol": { "en": "Official paired leaderboard, without skills; model and agent pairs remain distinct. Uses pass rate, not normalized gain.", "zh": "官方配对评测榜单；不使用 Skills；分别记录模型与 Agent。使用通过率，不使用归一化增益。" }, "source": { "name": "BenchFlow", "benchmark_creator": "BenchFlow", "url": "https://www.skillsbench.ai/", "data_url": "https://www.skillsbench.ai/", "sha256": "956ab5a9313b10e5e10fb23ff5ac1c172dad5a1acb8da34af454508444f2e82b", "retrieved_at": "2026-11-02T03:36:56.496569+00:00", "selector": "paired leaderboard table: Without Skills", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "40aa407c069b51b944d8", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: OpenHands; skills=false", "score": 0.515, "raw_score": 51.5, "raw_scale": 100, "source_record": "GPT-5.5|Agent: OpenHands; skills=false" }, { "id": "24e13f69b44f8740ed2c", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: Codex; skills=false", "score": 0.46799999999999997, "raw_score": 46.8, "raw_scale": 100, "source_record": "GPT-5.5|Agent: Codex; skills=false" }, { "id": "c9fc5b3e0a287dd5f9c0", "model": "Opus 4.8", "organization": "Claude", "configuration": "Agent: OpenHands; skills=false", "score": 0.457, "raw_score": 45.7, "raw_scale": 100, "source_record": "Opus 4.8|Agent: OpenHands; skills=false" }, { "id": "b36f377e22ebef9fa3c6", "model": "Opus 4.7", "organization": "Claude", "configuration": "Agent: Claude Code; skills=false", "score": 0.43, "raw_score": 43.0, "raw_scale": 100, "source_record": "Opus 4.7|Agent: Claude Code; skills=false" }, { "id": "036c952df59c5fd8a88f", "model": "Opus 4.7", "organization": "Claude", "configuration": "Agent: OpenHands; skills=false", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "Opus 4.7|Agent: OpenHands; skills=false" }, { "id": "0f65be2e4d55321911c9", "model": "Gemini 3.5 Flash", "organization": "Gemini", "configuration": "Agent: OpenHands; skills=false", "score": 0.41100000000000003, "raw_score": 41.1, "raw_scale": 100, "source_record": "Gemini 3.5 Flash|Agent: OpenHands; skills=false" }, { "id": "1655063d61852a54ef68", "model": "Gemini 3.1 Pro", "organization": "Gemini", "configuration": "Agent: Gemini CLI; skills=false", "score": 0.36, "raw_score": 36.0, "raw_scale": 100, "source_record": "Gemini 3.1 Pro|Agent: Gemini CLI; skills=false" }, { "id": "4a34023c2a7d05e23ded", "model": "Gemini 3 Flash", "organization": "Gemini", "configuration": "Agent: Gemini CLI; skills=false", "score": 0.342, "raw_score": 34.2, "raw_scale": 100, "source_record": "Gemini 3 Flash|Agent: Gemini CLI; skills=false" }, { "id": "afa646639b380356657a", "model": "Gemini 3.1 Pro", "organization": "Gemini", "configuration": "Agent: OpenHands; skills=false", "score": 0.33799999999999997, "raw_score": 33.8, "raw_scale": 100, "source_record": "Gemini 3.1 Pro|Agent: OpenHands; skills=false" }, { "id": "933f77304d495113d266", "model": "Opus 4.6", "organization": "Claude", "configuration": "Agent: Claude Code; skills=false", "score": 0.337, "raw_score": 33.7, "raw_scale": 100, "source_record": "Opus 4.6|Agent: Claude Code; skills=false" }, { "id": "6459392450aa0a68570d", "model": "Sonnet 4.6", "organization": "Claude", "configuration": "Agent: OpenHands; skills=false", "score": 0.335, "raw_score": 33.5, "raw_scale": 100, "source_record": "Sonnet 4.6|Agent: OpenHands; skills=false" }, { "id": "831b267fedd866c724e5", "model": "Kimi K2.6", "organization": "MoonshotAI", "configuration": "Agent: OpenHands; skills=false", "score": 0.33399999999999996, "raw_score": 33.4, "raw_scale": 100, "source_record": "Kimi K2.6|Agent: OpenHands; skills=false" }, { "id": "1e407cd2c2979cdd35c3", "model": "GLM 5.1", "organization": "Z.ai", "configuration": "Agent: OpenHands; skills=false", "score": 0.327, "raw_score": 32.7, "raw_scale": 100, "source_record": "GLM 5.1|Agent: OpenHands; skills=false" }, { "id": "11b69ac96fdc7c971e2a", "model": "GPT-5.4 Mini", "organization": "OpenAI", "configuration": "Agent: OpenHands; skills=false", "score": 0.299, "raw_score": 29.9, "raw_scale": 100, "source_record": "GPT-5.4 Mini|Agent: OpenHands; skills=false" }, { "id": "a15491d290a9d9b076e4", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Agent: Codex; skills=false", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "GPT-5.2|Agent: Codex; skills=false" }, { "id": "ee0eff40b107c697ae77", "model": "MiniMax M3", "organization": "Minimax", "configuration": "Agent: OpenHands; skills=false", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "MiniMax M3|Agent: OpenHands; skills=false" }, { "id": "912c3d1e35a1edb9db44", "model": "DeepSeek V4 Flash", "organization": "DeepSeek", "configuration": "Agent: OpenHands; skills=false", "score": 0.275, "raw_score": 27.5, "raw_scale": 100, "source_record": "DeepSeek V4 Flash|Agent: OpenHands; skills=false" }, { "id": "306f3afa8925134bb96d", "model": "DeepSeek V4 Pro", "organization": "DeepSeek", "configuration": "Agent: OpenHands; skills=false", "score": 0.26899999999999996, "raw_score": 26.9, "raw_scale": 100, "source_record": "DeepSeek V4 Pro|Agent: OpenHands; skills=false" }, { "id": "db8624ff85034c7a6019", "model": "Opus 4.5", "organization": "Claude", "configuration": "Agent: Claude Code; skills=false", "score": 0.23800000000000002, "raw_score": 23.8, "raw_scale": 100, "source_record": "Opus 4.5|Agent: Claude Code; skills=false" }, { "id": "68004b9a9649b71d5d96", "model": "Grok 4.3", "organization": "Grok", "configuration": "Agent: OpenHands; skills=false", "score": 0.228, "raw_score": 22.8, "raw_scale": 100, "source_record": "Grok 4.3|Agent: OpenHands; skills=false" }, { "id": "df860cf4fc22c4c4b7b1", "model": "MiniMax M2.7", "organization": "Minimax", "configuration": "Agent: OpenHands; skills=false", "score": 0.18100000000000002, "raw_score": 18.1, "raw_scale": 100, "source_record": "MiniMax M2.7|Agent: OpenHands; skills=false" }, { "id": "b03bab2cea2c197fc136", "model": "Sonnet 4.5", "organization": "Claude", "configuration": "Agent: Claude Code; skills=false", "score": 0.16699999999999998, "raw_score": 16.7, "raw_scale": 100, "source_record": "Sonnet 4.5|Agent: Claude Code; skills=false" }, { "id": "265ed722eadfb538e4fc", "model": "Gemini 3.1 Flash Lite", "organization": "Gemini", "configuration": "Agent: OpenHands; skills=false", "score": 0.16, "raw_score": 16.0, "raw_scale": 100, "source_record": "Gemini 3.1 Flash Lite|Agent: OpenHands; skills=false" }, { "id": "8f6ac0696c95cbfed23b", "model": "Haiku 4.5", "organization": "Claude", "configuration": "Agent: Claude Code; skills=false", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "Haiku 4.5|Agent: Claude Code; skills=false" } ] }, { "id": "finance-agent-v2-1", "name": "Finance Agent v2.1", "group": "专业工作", "unit": "score", "metric": "Overall rubric score", "protocol": { "en": "Reviewed FAB v2.1 dataset and FAB v2 harness; overall published rubric credit on 0–100.", "zh": "审校后的 FAB v2.1 数据集与 FAB v2 Agent；总体量规得分，保留 0–100。" }, "source": { "name": "Vals AI", "benchmark_creator": "Vals AI", "url": "https://www.vals.ai/benchmarks/fabv2-1", "data_url": "https://www.vals.ai/benchmarks/fabv2-1", "sha256": "bb07c726d952214d3099bd5fc54607f97aa0e25de754345ca1b1edf437ad3f02", "retrieved_at": "2026-11-02T03:35:10.107987+00:00", "selector": "First semantic leaderboard table: published Model and Accuracy/Task Pass Rate; protocol version from Astro benchmarkView.metadata", "kind": "evaluator", "status": "current_snapshot" }, "results": [ { "id": "a3da2de377a97bebc8d6", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Agent: FAB v2", "score": 67.85, "raw_score": 67.85, "raw_scale": 1, "source_record": "anthropic_claude-opus-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-opus-5-5", "source_date": "2026-10-29" }, { "id": "60411aab53877cb6fe7d", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: FAB v2", "score": 67.66, "raw_score": 67.66, "raw_scale": 1, "source_record": "google_gemini-4-argon", "report_url": "https://www.vals.ai/models/google_gemini-4-argon", "source_date": "2026-10-29" }, { "id": "7fda76500302bad280b4", "model": "Wonder Full", "organization": "Meta", "configuration": "Agent: FAB v2", "score": 67.58, "raw_score": 67.58, "raw_scale": 1, "source_record": "meta_muse-wonder-full", "report_url": "https://www.vals.ai/models/meta_muse-wonder-full", "source_date": "2026-10-29" }, { "id": "0bed8fcf5e0943770bf8", "model": "Claude Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: FAB v2", "score": 67.28, "raw_score": 67.28, "raw_scale": 1, "source_record": "anthropic_claude-sonnet-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-sonnet-5-5", "source_date": "2026-10-29" }, { "id": "d9bdb252d688c4b0cf9b", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Agent: FAB v2", "score": 67.16, "raw_score": 67.16, "raw_scale": 1, "source_record": "google_gemini-4-carbon", "report_url": "https://www.vals.ai/models/google_gemini-4-carbon", "source_date": "2026-10-29" }, { "id": "827ff8d39831f41c2628", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: FAB v2", "score": 65.76, "raw_score": 65.76, "raw_scale": 1, "source_record": "google_gemini-3.8-flash", "report_url": "https://www.vals.ai/models/google_gemini-3.8-flash", "source_date": "2026-10-29" }, { "id": "39bf7a5189ac794c25eb", "model": "Wonder", "organization": "Meta", "configuration": "Agent: FAB v2", "score": 65.55, "raw_score": 65.55, "raw_scale": 1, "source_record": "meta_muse-wonder", "report_url": "https://www.vals.ai/models/meta_muse-wonder", "source_date": "2026-10-29" }, { "id": "a915ad771463af443fa7", "model": "DeepSeek V4.1 Flash", "organization": "DeepSeek", "configuration": "Agent: FAB v2", "score": 65.43, "raw_score": 65.43, "raw_scale": 1, "source_record": "deepseek_deepseek-v4.1-flash", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4.1-flash", "source_date": "2026-10-29" }, { "id": "eec80ad9a229ca24d7a4", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: FAB v2", "score": 65.37, "raw_score": 65.37, "raw_scale": 1, "source_record": "anthropic_claude-haiku-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-haiku-5-5", "source_date": "2026-10-29" }, { "id": "4bd6a9fbb7a7f3f6b3cb", "model": "GLM 5.3", "organization": "Z AI", "configuration": "Agent: FAB v2", "score": 65.0, "raw_score": 65.0, "raw_scale": 1, "source_record": "zai_glm-5.3", "report_url": "https://www.vals.ai/models/zai_glm-5.3", "source_date": "2026-10-29" }, { "id": "27ea1414ab5e1c3545f0", "model": "MiMo V2.6 Pro", "organization": "Xiaomi", "configuration": "Agent: FAB v2", "score": 64.26, "raw_score": 64.26, "raw_scale": 1, "source_record": "xiaomi_mimo-v2.6-pro", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.6-pro", "source_date": "2026-10-29" }, { "id": "bf64943a685862f5e29c", "model": "MiMo V2.6 Flash", "organization": "Xiaomi", "configuration": "Agent: FAB v2", "score": 63.32, "raw_score": 63.32, "raw_scale": 1, "source_record": "xiaomi_mimo-v2.6-flash", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.6-flash", "source_date": "2026-10-29" }, { "id": "66cd326c12bf70fc64a0", "model": "DeepSeek V4.1 Flash 1017", "organization": "DeepSeek", "configuration": "Agent: FAB v2", "score": 62.93, "raw_score": 62.93, "raw_scale": 1, "source_record": "deepseek_deepseek-v4.1-flash-1017", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4.1-flash-1017", "source_date": "2026-10-29" }, { "id": "aeecf45f59048f90205f", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Agent: FAB v2", "score": 61.82, "raw_score": 61.82, "raw_scale": 1, "source_record": "openai_gpt-6.1-sol", "report_url": "https://www.vals.ai/models/openai_gpt-6.1-sol", "source_date": "2026-10-29" }, { "id": "d83ab577bbc4c7df82d2", "model": "Mistral Large 4 OSS", "organization": "Mistral AI", "configuration": "Agent: FAB v2", "score": 61.72, "raw_score": 61.72, "raw_scale": 1, "source_record": "mistralai_mistral-large-4-oss", "report_url": "https://www.vals.ai/models/mistralai_mistral-large-4-oss", "source_date": "2026-10-29" }, { "id": "b9a2ce55f2b29b7654e3", "model": "Hy4 Preview", "organization": "Tencent", "configuration": "Agent: FAB v2", "score": 61.62, "raw_score": 61.62, "raw_scale": 1, "source_record": "tencent_hy4-preview", "report_url": "https://www.vals.ai/models/tencent_hy4-preview", "source_date": "2026-10-29" }, { "id": "27c8de65645a6ca27e33", "model": "Kimi K3.1 Pro", "organization": "Moonshot AI", "configuration": "Agent: FAB v2", "score": 61.49, "raw_score": 61.49, "raw_scale": 1, "source_record": "kimi_kimi-k3.1-pro", "report_url": "https://www.vals.ai/models/kimi_kimi-k3.1-pro", "source_date": "2026-10-29" }, { "id": "95766b3eed6c8efdf078", "model": "Mistral Large 4", "organization": "Mistral AI", "configuration": "Agent: FAB v2", "score": 61.38, "raw_score": 61.38, "raw_scale": 1, "source_record": "mistralai_mistral-large-4", "report_url": "https://www.vals.ai/models/mistralai_mistral-large-4", "source_date": "2026-10-29" }, { "id": "f03629565611bd73920f", "model": "Muse Spark Open", "organization": "Meta", "configuration": "Agent: FAB v2", "score": 60.66, "raw_score": 60.66, "raw_scale": 1, "source_record": "meta_muse-spark-open", "report_url": "https://www.vals.ai/models/meta_muse-spark-open", "source_date": "2026-10-29" }, { "id": "35001553257750ea3f73", "model": "GLM 5.3 Flash", "organization": "Z AI", "configuration": "Agent: FAB v2", "score": 60.59, "raw_score": 60.59, "raw_scale": 1, "source_record": "zai_glm-5.3-flash", "report_url": "https://www.vals.ai/models/zai_glm-5.3-flash", "source_date": "2026-10-29" }, { "id": "681a52f4091b0b196f6d", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: FAB v2", "score": 60.59, "raw_score": 60.59, "raw_scale": 1, "source_record": "grok_grok-4.7", "report_url": "https://www.vals.ai/models/grok_grok-4.7", "source_date": "2026-10-29" }, { "id": "79cf2170a4702bd586f7", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: FAB v2", "score": 60.43, "raw_score": 60.43, "raw_scale": 1, "source_record": "openai_gpt-6-astra", "report_url": "https://www.vals.ai/models/openai_gpt-6-astra", "source_date": "2026-10-29" }, { "id": "40a7d16e2073321233f5", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Agent: FAB v2", "score": 60.01, "raw_score": 60.01, "raw_scale": 1, "source_record": "kimi_kimi-k3", "report_url": "https://www.vals.ai/models/kimi_kimi-k3", "source_date": "2026-10-29" }, { "id": "abc13805dd0f035befd0", "model": "Step 5 Preview", "organization": "StepFun", "configuration": "Agent: FAB v2", "score": 58.9, "raw_score": 58.9, "raw_scale": 1, "source_record": "stepfun_step-5-preview", "report_url": "https://www.vals.ai/models/stepfun_step-5-preview", "source_date": "2026-10-29" }, { "id": "37315702c9531c6b5cdc", "model": "MiniMax-M3.1-Flash", "organization": "MiniMax", "configuration": "Agent: FAB v2", "score": 58.83, "raw_score": 58.83, "raw_scale": 1, "source_record": "minimax_MiniMax-M3.1-flash", "report_url": "https://www.vals.ai/models/minimax_MiniMax-M3.1-flash", "source_date": "2026-10-29" }, { "id": "36834d64ad7631afe4f7", "model": "Qwen 3.8 Max 0902", "organization": "Alibaba", "configuration": "Agent: FAB v2", "score": 58.83, "raw_score": 58.83, "raw_scale": 1, "source_record": "alibaba_qwen3.8-max-0902", "report_url": "https://www.vals.ai/models/alibaba_qwen3.8-max-0902", "source_date": "2026-10-29" }, { "id": "598b6ad823b01e30feae", "model": "DeepSeek V4 Pro 0813", "organization": "DeepSeek", "configuration": "Agent: FAB v2", "score": 57.12, "raw_score": 57.12, "raw_scale": 1, "source_record": "deepseek_deepseek-v4-pro-0813", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4-pro-0813", "source_date": "2026-10-29" }, { "id": "b87ee951a6446969edad", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Agent: FAB v2", "score": 56.84, "raw_score": 56.84, "raw_scale": 1, "source_record": "openai_gpt-6-luna", "report_url": "https://www.vals.ai/models/openai_gpt-6-luna", "source_date": "2026-10-29" }, { "id": "5935a65d2d9d259a42a2", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "Agent: FAB v2", "score": 55.96, "raw_score": 55.96, "raw_scale": 1, "source_record": "openai_gpt-6.1-terra", "report_url": "https://www.vals.ai/models/openai_gpt-6.1-terra", "source_date": "2026-10-29" }, { "id": "5991cfd5874e85410591", "model": "Mercury 3", "organization": "Inception", "configuration": "Agent: FAB v2", "score": 54.71, "raw_score": 54.71, "raw_scale": 1, "source_record": "inception_mercury-3", "report_url": "https://www.vals.ai/models/inception_mercury-3", "source_date": "2026-10-29" }, { "id": "eb1e2e3eecb3295b2be2", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "Agent: FAB v2", "score": 54.34, "raw_score": 54.34, "raw_scale": 1, "source_record": "minimax_MiniMax-M3", "report_url": "https://www.vals.ai/models/minimax_MiniMax-M3", "source_date": "2026-10-29" }, { "id": "a6d04c1a48e6b396d39a", "model": "Beam 1.0", "organization": "Reflection", "configuration": "Agent: FAB v2", "score": 51.83, "raw_score": 51.83, "raw_scale": 1, "source_record": "reflection_Beam-1.0", "report_url": "https://www.vals.ai/models/reflection_Beam-1.0", "source_date": "2026-10-29" }, { "id": "a018d28301ef0ec44ff7", "model": "Inkling", "organization": "Thinking Machines Lab", "configuration": "Agent: FAB v2", "score": 49.87, "raw_score": 49.87, "raw_scale": 1, "source_record": "thinkingmachines_inkling", "report_url": "https://www.vals.ai/models/thinkingmachines_inkling", "source_date": "2026-10-29" }, { "id": "733d417d4f8697862f6c", "model": "Nemotron 3.5 Super VL", "organization": "", "configuration": "Agent: FAB v2", "score": 47.73, "raw_score": 47.73, "raw_scale": 1, "source_record": "baseten_nemotron-3.5-super-vl-120b-a12b", "report_url": "https://www.vals.ai/models/baseten_nemotron-3.5-super-vl-120b-a12b", "source_date": "2026-10-29" }, { "id": "fdc3298cc11c98689d04", "model": "Inkling Small", "organization": "Thinking Machines Lab", "configuration": "Agent: FAB v2", "score": 44.78, "raw_score": 44.78, "raw_scale": 1, "source_record": "thinkingmachines_inkling-small", "report_url": "https://www.vals.ai/models/thinkingmachines_inkling-small", "source_date": "2026-10-29" } ], "score_max": 100 }, { "id": "tax-agent-bench", "name": "Tax Agent Bench", "group": "专业工作", "unit": "%", "metric": "All-pass accuracy", "protocol": { "en": "Research-grade US corporate tax questions; all-pass accuracy under the published Tax Agent Bench protocol.", "zh": "研究级美国企业税务问题；按 Tax Agent Bench 官方协议计算完整通过率。" }, "source": { "name": "Vals AI", "benchmark_creator": "Vals AI", "url": "https://www.vals.ai/benchmarks/tax_agent_bench", "data_url": "https://www.vals.ai/benchmarks/tax_agent_bench", "sha256": "bf38e44de908e8510565bfde4b7ec70f800ae0dad6b01eb3cde7d060a48cf736", "retrieved_at": "2026-11-02T03:35:10.229671+00:00", "selector": "First semantic leaderboard table: published Model and Accuracy/Task Pass Rate; protocol version from Astro benchmarkView.metadata", "kind": "evaluator", "status": "current_snapshot" }, "results": [ { "id": "d9bdb252d688c4b0cf9b", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.5178, "raw_score": 51.78, "raw_scale": 100, "source_record": "google_gemini-4-carbon", "report_url": "https://www.vals.ai/models/google_gemini-4-carbon", "source_date": "2026-10-30" }, { "id": "36834d64ad7631afe4f7", "model": "Qwen 3.8 Max 0902", "organization": "Alibaba", "configuration": "Agent: Tax Agent Bench", "score": 0.4966, "raw_score": 49.66, "raw_scale": 100, "source_record": "alibaba_qwen3.8-max-0902", "report_url": "https://www.vals.ai/models/alibaba_qwen3.8-max-0902", "source_date": "2026-10-30" }, { "id": "d42a8daae41007610b1c", "model": "Claude Fable 5.1", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.4919, "raw_score": 49.19, "raw_scale": 100, "source_record": "anthropic_claude-fable-5-1", "report_url": "https://www.vals.ai/models/anthropic_claude-fable-5-1", "source_date": "2026-10-30" }, { "id": "a3da2de377a97bebc8d6", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.4557, "raw_score": 45.57, "raw_scale": 100, "source_record": "anthropic_claude-opus-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-opus-5-5", "source_date": "2026-10-30" }, { "id": "ab8e6ceab76f1c7317ef", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.45549999999999996, "raw_score": 45.55, "raw_scale": 100, "source_record": "anthropic_claude-opus-5", "report_url": "https://www.vals.ai/models/anthropic_claude-opus-5", "source_date": "2026-10-30" }, { "id": "60411aab53877cb6fe7d", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.4456, "raw_score": 44.56, "raw_scale": 100, "source_record": "google_gemini-4-argon", "report_url": "https://www.vals.ai/models/google_gemini-4-argon", "source_date": "2026-10-30" }, { "id": "66cd326c12bf70fc64a0", "model": "DeepSeek V4.1 Flash 1017", "organization": "DeepSeek", "configuration": "Agent: Tax Agent Bench", "score": 0.4435, "raw_score": 44.35, "raw_scale": 100, "source_record": "deepseek_deepseek-v4.1-flash-1017", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4.1-flash-1017", "source_date": "2026-10-30" }, { "id": "7fda76500302bad280b4", "model": "Wonder Full", "organization": "Meta", "configuration": "Agent: Tax Agent Bench", "score": 0.43, "raw_score": 43.0, "raw_scale": 100, "source_record": "meta_muse-wonder-full", "report_url": "https://www.vals.ai/models/meta_muse-wonder-full", "source_date": "2026-10-30" }, { "id": "b00156160535fcd84dc7", "model": "Muse Spark 1.3 Max", "organization": "Meta", "configuration": "Agent: Tax Agent Bench", "score": 0.4244, "raw_score": 42.44, "raw_scale": 100, "source_record": "meta_muse_spark_1_3_max", "report_url": "https://www.vals.ai/models/meta_muse_spark_1_3_max", "source_date": "2026-10-30" }, { "id": "0bed8fcf5e0943770bf8", "model": "Claude Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.41960000000000003, "raw_score": 41.96, "raw_scale": 100, "source_record": "anthropic_claude-sonnet-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-sonnet-5-5", "source_date": "2026-10-30" }, { "id": "4bd6a9fbb7a7f3f6b3cb", "model": "GLM 5.3", "organization": "Z AI", "configuration": "Agent: Tax Agent Bench", "score": 0.397, "raw_score": 39.7, "raw_scale": 100, "source_record": "zai_glm-5.3", "report_url": "https://www.vals.ai/models/zai_glm-5.3", "source_date": "2026-10-30" }, { "id": "fb93b6a8154e7adc1f3d", "model": "Claude Fable 5", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.38280000000000003, "raw_score": 38.28, "raw_scale": 100, "source_record": "anthropic_claude-fable-5", "report_url": "https://www.vals.ai/models/anthropic_claude-fable-5", "source_date": "2026-10-30" }, { "id": "39bf7a5189ac794c25eb", "model": "Wonder", "organization": "Meta", "configuration": "Agent: Tax Agent Bench", "score": 0.3627, "raw_score": 36.27, "raw_scale": 100, "source_record": "meta_muse-wonder", "report_url": "https://www.vals.ai/models/meta_muse-wonder", "source_date": "2026-10-30" }, { "id": "6d08409d8754a30eafa1", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: Tax Agent Bench", "score": 0.35969999999999996, "raw_score": 35.97, "raw_scale": 100, "source_record": "meta_muse_spark_1_3", "report_url": "https://www.vals.ai/models/meta_muse_spark_1_3", "source_date": "2026-10-30" }, { "id": "681a52f4091b0b196f6d", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: Tax Agent Bench", "score": 0.342, "raw_score": 34.2, "raw_scale": 100, "source_record": "grok_grok-4.7", "report_url": "https://www.vals.ai/models/grok_grok-4.7", "source_date": "2026-10-30" }, { "id": "40a7d16e2073321233f5", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Agent: Tax Agent Bench", "score": 0.3337, "raw_score": 33.37, "raw_scale": 100, "source_record": "kimi_kimi-k3", "report_url": "https://www.vals.ai/models/kimi_kimi-k3", "source_date": "2026-10-30" }, { "id": "d536d5bb5d7ecd5b0923", "model": "Grok 4.6", "organization": "xAI", "configuration": "Agent: Tax Agent Bench", "score": 0.33140000000000003, "raw_score": 33.14, "raw_scale": 100, "source_record": "grok_grok-4.6", "report_url": "https://www.vals.ai/models/grok_grok-4.6", "source_date": "2026-10-30" }, { "id": "8eb158af196e8a4f4352", "model": "Muse Spark 1.2", "organization": "Meta", "configuration": "Agent: Tax Agent Bench", "score": 0.32799999999999996, "raw_score": 32.8, "raw_scale": 100, "source_record": "meta_muse_spark_1_2", "report_url": "https://www.vals.ai/models/meta_muse_spark_1_2", "source_date": "2026-10-30" }, { "id": "b9a2ce55f2b29b7654e3", "model": "Hy4 Preview", "organization": "Tencent", "configuration": "Agent: Tax Agent Bench", "score": 0.3262, "raw_score": 32.62, "raw_scale": 100, "source_record": "tencent_hy4-preview", "report_url": "https://www.vals.ai/models/tencent_hy4-preview", "source_date": "2026-10-30" }, { "id": "27ea1414ab5e1c3545f0", "model": "MiMo V2.6 Pro", "organization": "Xiaomi", "configuration": "Agent: Tax Agent Bench", "score": 0.3239, "raw_score": 32.39, "raw_scale": 100, "source_record": "xiaomi_mimo-v2.6-pro", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.6-pro", "source_date": "2026-10-30" }, { "id": "37315702c9531c6b5cdc", "model": "MiniMax-M3.1-Flash", "organization": "MiniMax", "configuration": "Agent: Tax Agent Bench", "score": 0.3214, "raw_score": 32.14, "raw_scale": 100, "source_record": "minimax_MiniMax-M3.1-flash", "report_url": "https://www.vals.ai/models/minimax_MiniMax-M3.1-flash", "source_date": "2026-10-30" }, { "id": "827ff8d39831f41c2628", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.3212, "raw_score": 32.12, "raw_scale": 100, "source_record": "google_gemini-3.8-flash", "report_url": "https://www.vals.ai/models/google_gemini-3.8-flash", "source_date": "2026-10-30" }, { "id": "20c2d8ba0d7e810bb18a", "model": "Qwen 3.8 Max", "organization": "Alibaba", "configuration": "Agent: Tax Agent Bench", "score": 0.3209, "raw_score": 32.09, "raw_scale": 100, "source_record": "alibaba_qwen3.8-max", "report_url": "https://www.vals.ai/models/alibaba_qwen3.8-max", "source_date": "2026-10-30" }, { "id": "eec80ad9a229ca24d7a4", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.31920000000000004, "raw_score": 31.92, "raw_scale": 100, "source_record": "anthropic_claude-haiku-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-haiku-5-5", "source_date": "2026-10-30" }, { "id": "35001553257750ea3f73", "model": "GLM 5.3 Flash", "organization": "Z AI", "configuration": "Agent: Tax Agent Bench", "score": 0.3155, "raw_score": 31.55, "raw_scale": 100, "source_record": "zai_glm-5.3-flash", "report_url": "https://www.vals.ai/models/zai_glm-5.3-flash", "source_date": "2026-10-30" }, { "id": "fc8e214f91305e6e904c", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.3109, "raw_score": 31.09, "raw_scale": 100, "source_record": "openai_gpt-5.6-sol", "report_url": "https://www.vals.ai/models/openai_gpt-5.6-sol", "source_date": "2026-10-30" }, { "id": "27280a601e680f73ec35", "model": "Ember-1", "organization": "", "configuration": "Agent: Tax Agent Bench", "score": 0.31079999999999997, "raw_score": 31.08, "raw_scale": 100, "source_record": "fireworks_ember-1", "report_url": "https://www.vals.ai/models/fireworks_ember-1", "source_date": "2026-10-30" }, { "id": "9b9dbb84d1e16e569f35", "model": "Qwen 3.8 27B", "organization": "Alibaba", "configuration": "Agent: Tax Agent Bench", "score": 0.3089, "raw_score": 30.89, "raw_scale": 100, "source_record": "alibaba_qwen3.8-27b", "report_url": "https://www.vals.ai/models/alibaba_qwen3.8-27b", "source_date": "2026-10-30" }, { "id": "d83ab577bbc4c7df82d2", "model": "Mistral Large 4 OSS", "organization": "Mistral AI", "configuration": "Agent: Tax Agent Bench", "score": 0.30670000000000003, "raw_score": 30.67, "raw_scale": 100, "source_record": "mistralai_mistral-large-4-oss", "report_url": "https://www.vals.ai/models/mistralai_mistral-large-4-oss", "source_date": "2026-10-30" }, { "id": "95766b3eed6c8efdf078", "model": "Mistral Large 4", "organization": "Mistral AI", "configuration": "Agent: Tax Agent Bench", "score": 0.3039, "raw_score": 30.39, "raw_scale": 100, "source_record": "mistralai_mistral-large-4", "report_url": "https://www.vals.ai/models/mistralai_mistral-large-4", "source_date": "2026-10-30" }, { "id": "1c34f71d105176ad75b5", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.3004, "raw_score": 30.04, "raw_scale": 100, "source_record": "openai_gpt-5.6-terra", "report_url": "https://www.vals.ai/models/openai_gpt-5.6-terra", "source_date": "2026-10-30" }, { "id": "ef41eab79f05e4053300", "model": "Claude Opus 4.8", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.3, "raw_score": 30.0, "raw_scale": 100, "source_record": "anthropic_claude-opus-4-8", "report_url": "https://www.vals.ai/models/anthropic_claude-opus-4-8", "source_date": "2026-10-30" }, { "id": "0db143ddcb5f9f0cd296", "model": "Claude Sonnet 5", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.2937, "raw_score": 29.37, "raw_scale": 100, "source_record": "anthropic_claude-sonnet-5", "report_url": "https://www.vals.ai/models/anthropic_claude-sonnet-5", "source_date": "2026-10-30" }, { "id": "bf64943a685862f5e29c", "model": "MiMo V2.6 Flash", "organization": "Xiaomi", "configuration": "Agent: Tax Agent Bench", "score": 0.2851, "raw_score": 28.51, "raw_scale": 100, "source_record": "xiaomi_mimo-v2.6-flash", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.6-flash", "source_date": "2026-10-30" }, { "id": "8e33c2930002b606b5e9", "model": "DeepSeek V4 Flash 0731", "organization": "DeepSeek", "configuration": "Agent: Tax Agent Bench", "score": 0.2807, "raw_score": 28.07, "raw_scale": 100, "source_record": "deepseek_deepseek-v4-flash-0731", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4-flash-0731", "source_date": "2026-10-30" }, { "id": "abc13805dd0f035befd0", "model": "Step 5 Preview", "organization": "StepFun", "configuration": "Agent: Tax Agent Bench", "score": 0.2727, "raw_score": 27.27, "raw_scale": 100, "source_record": "stepfun_step-5-preview", "report_url": "https://www.vals.ai/models/stepfun_step-5-preview", "source_date": "2026-10-30" }, { "id": "598b6ad823b01e30feae", "model": "DeepSeek V4 Pro 0813", "organization": "DeepSeek", "configuration": "Agent: Tax Agent Bench", "score": 0.2676, "raw_score": 26.76, "raw_scale": 100, "source_record": "deepseek_deepseek-v4-pro-0813", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4-pro-0813", "source_date": "2026-10-30" }, { "id": "a915ad771463af443fa7", "model": "DeepSeek V4.1 Flash", "organization": "DeepSeek", "configuration": "Agent: Tax Agent Bench", "score": 0.2663, "raw_score": 26.63, "raw_scale": 100, "source_record": "deepseek_deepseek-v4.1-flash", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4.1-flash", "source_date": "2026-10-30" }, { "id": "f03629565611bd73920f", "model": "Muse Spark Open", "organization": "Meta", "configuration": "Agent: Tax Agent Bench", "score": 0.2632, "raw_score": 26.32, "raw_scale": 100, "source_record": "meta_muse-spark-open", "report_url": "https://www.vals.ai/models/meta_muse-spark-open", "source_date": "2026-10-30" }, { "id": "b01b1aa54437fe329bb8", "model": "Claude Sonnet 4.6", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.2584, "raw_score": 25.84, "raw_scale": 100, "source_record": "anthropic_claude-sonnet-4-6", "report_url": "https://www.vals.ai/models/anthropic_claude-sonnet-4-6", "source_date": "2026-10-30" }, { "id": "4540bd1f3e17a5c8ae1f", "model": "DeepSeek V4", "organization": "DeepSeek", "configuration": "Agent: Tax Agent Bench", "score": 0.25170000000000003, "raw_score": 25.17, "raw_scale": 100, "source_record": "deepseek_deepseek-v4-pro", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4-pro", "source_date": "2026-10-30" }, { "id": "27c8de65645a6ca27e33", "model": "Kimi K3.1 Pro", "organization": "Moonshot AI", "configuration": "Agent: Tax Agent Bench", "score": 0.2487, "raw_score": 24.87, "raw_scale": 100, "source_record": "kimi_kimi-k3.1-pro", "report_url": "https://www.vals.ai/models/kimi_kimi-k3.1-pro", "source_date": "2026-10-30" }, { "id": "258b3be79e49e400a5a2", "model": "Grok 4.5", "organization": "xAI", "configuration": "Agent: Tax Agent Bench", "score": 0.2486, "raw_score": 24.86, "raw_scale": 100, "source_record": "grok_grok-4.5", "report_url": "https://www.vals.ai/models/grok_grok-4.5", "source_date": "2026-10-30" }, { "id": "0beff6b902fddabeecd7", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.2482, "raw_score": 24.82, "raw_scale": 100, "source_record": "openai_gpt-5.6-luna", "report_url": "https://www.vals.ai/models/openai_gpt-5.6-luna", "source_date": "2026-10-30" }, { "id": "37c05d39fdc6bf19735c", "model": "Claude Opus 4.7", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.23739999999999997, "raw_score": 23.74, "raw_scale": 100, "source_record": "anthropic_claude-opus-4-7", "report_url": "https://www.vals.ai/models/anthropic_claude-opus-4-7", "source_date": "2026-10-30" }, { "id": "0344348617332d30c847", "model": "Gemini 3.7 Flash", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.228, "raw_score": 22.8, "raw_scale": 100, "source_record": "google_gemini-3.7-flash", "report_url": "https://www.vals.ai/models/google_gemini-3.7-flash", "source_date": "2026-10-30" }, { "id": "804175f6150b13b80691", "model": "GPT 5.5", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.21760000000000002, "raw_score": 21.76, "raw_scale": 100, "source_record": "openai_gpt-5.5", "report_url": "https://www.vals.ai/models/openai_gpt-5.5", "source_date": "2026-10-30" }, { "id": "aeecf45f59048f90205f", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.2175, "raw_score": 21.75, "raw_scale": 100, "source_record": "openai_gpt-6.1-sol", "report_url": "https://www.vals.ai/models/openai_gpt-6.1-sol", "source_date": "2026-10-30" }, { "id": "be88ea0f1a2f6a7a72f4", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.2165, "raw_score": 21.65, "raw_scale": 100, "source_record": "google_gemini-3.5-flash", "report_url": "https://www.vals.ai/models/google_gemini-3.5-flash", "source_date": "2026-10-30" }, { "id": "79cf2170a4702bd586f7", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.2072, "raw_score": 20.72, "raw_scale": 100, "source_record": "openai_gpt-6-astra", "report_url": "https://www.vals.ai/models/openai_gpt-6-astra", "source_date": "2026-10-30" }, { "id": "a8d29d022d536ba6e0cf", "model": "Ling 3.0 Flash Fin", "organization": "Ant Group", "configuration": "Agent: Tax Agent Bench", "score": 0.20600000000000002, "raw_score": 20.6, "raw_scale": 100, "source_record": "ant_ling-3.0-flash-fin", "report_url": "https://www.vals.ai/models/ant_ling-3.0-flash-fin", "source_date": "2026-10-30" }, { "id": "eb1e2e3eecb3295b2be2", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "Agent: Tax Agent Bench", "score": 0.2059, "raw_score": 20.59, "raw_scale": 100, "source_record": "minimax_MiniMax-M3", "report_url": "https://www.vals.ai/models/minimax_MiniMax-M3", "source_date": "2026-10-30" }, { "id": "b87ee951a6446969edad", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.19949999999999998, "raw_score": 19.95, "raw_scale": 100, "source_record": "openai_gpt-6-luna", "report_url": "https://www.vals.ai/models/openai_gpt-6-luna", "source_date": "2026-10-30" }, { "id": "5991cfd5874e85410591", "model": "Mercury 3", "organization": "Inception", "configuration": "Agent: Tax Agent Bench", "score": 0.18170000000000003, "raw_score": 18.17, "raw_scale": 100, "source_record": "inception_mercury-3", "report_url": "https://www.vals.ai/models/inception_mercury-3", "source_date": "2026-10-30" }, { "id": "8a4e47d7f4b1e7651534", "model": "Gemini 3.6 Flash", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.1804, "raw_score": 18.04, "raw_scale": 100, "source_record": "google_gemini-3.6-flash", "report_url": "https://www.vals.ai/models/google_gemini-3.6-flash", "source_date": "2026-10-30" }, { "id": "5935a65d2d9d259a42a2", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.17079999999999998, "raw_score": 17.08, "raw_scale": 100, "source_record": "openai_gpt-6.1-terra", "report_url": "https://www.vals.ai/models/openai_gpt-6.1-terra", "source_date": "2026-10-30" }, { "id": "d6afa805e3942b2c24f5", "model": "Qwen 3.7 Max", "organization": "Alibaba", "configuration": "Agent: Tax Agent Bench", "score": 0.1583, "raw_score": 15.83, "raw_scale": 100, "source_record": "alibaba_qwen3.7-max", "report_url": "https://www.vals.ai/models/alibaba_qwen3.7-max", "source_date": "2026-10-30" }, { "id": "fdc3298cc11c98689d04", "model": "Inkling Small", "organization": "Thinking Machines Lab", "configuration": "Agent: Tax Agent Bench", "score": 0.1539, "raw_score": 15.39, "raw_scale": 100, "source_record": "thinkingmachines_inkling-small", "report_url": "https://www.vals.ai/models/thinkingmachines_inkling-small", "source_date": "2026-10-30" }, { "id": "a018d28301ef0ec44ff7", "model": "Inkling", "organization": "Thinking Machines Lab", "configuration": "Agent: Tax Agent Bench", "score": 0.15289999999999998, "raw_score": 15.29, "raw_scale": 100, "source_record": "thinkingmachines_inkling", "report_url": "https://www.vals.ai/models/thinkingmachines_inkling", "source_date": "2026-10-30" }, { "id": "e5fa761a0d228737d119", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.15, "raw_score": 15.0, "raw_scale": 100, "source_record": "openai_gpt-6-sol", "report_url": "https://www.vals.ai/models/openai_gpt-6-sol", "source_date": "2026-10-30" }, { "id": "cbc0969d0bf3dbbb3d47", "model": "Kimi K2.6", "organization": "Moonshot AI", "configuration": "Agent: Tax Agent Bench", "score": 0.124, "raw_score": 12.4, "raw_scale": 100, "source_record": "kimi_kimi-k2.6", "report_url": "https://www.vals.ai/models/kimi_kimi-k2.6", "source_date": "2026-10-30" }, { "id": "a8385e3057e4f2a90f0b", "model": "Gemini 3 Flash (12/25)", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.1184, "raw_score": 11.84, "raw_scale": 100, "source_record": "google_gemini-3-flash-preview", "report_url": "https://www.vals.ai/models/google_gemini-3-flash-preview", "source_date": "2026-10-30" }, { "id": "733d417d4f8697862f6c", "model": "Nemotron 3.5 Super VL", "organization": "", "configuration": "Agent: Tax Agent Bench", "score": 0.1155, "raw_score": 11.55, "raw_scale": 100, "source_record": "baseten_nemotron-3.5-super-vl-120b-a12b", "report_url": "https://www.vals.ai/models/baseten_nemotron-3.5-super-vl-120b-a12b", "source_date": "2026-10-30" }, { "id": "9326aff976d061071f02", "model": "MiMo V2.5 Pro", "organization": "Xiaomi", "configuration": "Agent: Tax Agent Bench", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "xiaomi_mimo-v2.5-pro", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.5-pro", "source_date": "2026-10-30" }, { "id": "f59c6e6de284ffd3bcdc", "model": "Qwen 3.7 Plus", "organization": "Alibaba", "configuration": "Agent: Tax Agent Bench", "score": 0.1047, "raw_score": 10.47, "raw_scale": 100, "source_record": "alibaba_qwen3.7-plus", "report_url": "https://www.vals.ai/models/alibaba_qwen3.7-plus", "source_date": "2026-10-30" }, { "id": "7241c51cf41906abc4b2", "model": "Gemini 3.1 Pro Preview (02/26)", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.0929, "raw_score": 9.29, "raw_scale": 100, "source_record": "google_gemini-3.1-pro-preview", "report_url": "https://www.vals.ai/models/google_gemini-3.1-pro-preview", "source_date": "2026-10-30" }, { "id": "a6d04c1a48e6b396d39a", "model": "Beam 1.0", "organization": "Reflection", "configuration": "Agent: Tax Agent Bench", "score": 0.08810000000000001, "raw_score": 8.81, "raw_scale": 100, "source_record": "reflection_Beam-1.0", "report_url": "https://www.vals.ai/models/reflection_Beam-1.0", "source_date": "2026-10-30" }, { "id": "66acf9bfdfac8004c063", "model": "GPT 5.4 Mini", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.08810000000000001, "raw_score": 8.81, "raw_scale": 100, "source_record": "openai_gpt-5.4-mini-2026-03-17", "report_url": "https://www.vals.ai/models/openai_gpt-5.4-mini-2026-03-17", "source_date": "2026-10-30" }, { "id": "4ac6ec3c3f898a2b7824", "model": "MiMo V2.5", "organization": "Xiaomi", "configuration": "Agent: Tax Agent Bench", "score": 0.07769999999999999, "raw_score": 7.77, "raw_scale": 100, "source_record": "xiaomi_mimo-v2.5", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.5", "source_date": "2026-10-30" }, { "id": "4c2a1b5cf509c714299b", "model": "Qwen 3.6 Plus", "organization": "Alibaba", "configuration": "Agent: Tax Agent Bench", "score": 0.0768, "raw_score": 7.68, "raw_scale": 100, "source_record": "alibaba_qwen3.6-plus", "report_url": "https://www.vals.ai/models/alibaba_qwen3.6-plus", "source_date": "2026-10-30" }, { "id": "a33ec0d655df90ed4f45", "model": "Gemini 3.5 Flash Lite", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.0725, "raw_score": 7.25, "raw_scale": 100, "source_record": "google_gemini-3.5-flash-lite", "report_url": "https://www.vals.ai/models/google_gemini-3.5-flash-lite", "source_date": "2026-10-30" }, { "id": "0b6145e88e2beb9cfff0", "model": "Grok 4.20 (Reasoning)", "organization": "xAI", "configuration": "Agent: Tax Agent Bench", "score": 0.0695, "raw_score": 6.95, "raw_scale": 100, "source_record": "grok_grok-4.20-0309-reasoning", "report_url": "https://www.vals.ai/models/grok_grok-4.20-0309-reasoning", "source_date": "2026-10-30" }, { "id": "b4f7d467d1f0a6118eea", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "Agent: Tax Agent Bench", "score": 0.0687, "raw_score": 6.87, "raw_scale": 100, "source_record": "minimax_MiniMax-M2.7", "report_url": "https://www.vals.ai/models/minimax_MiniMax-M2.7", "source_date": "2026-10-30" }, { "id": "c37178751c0b89453013", "model": "Grok 4.3", "organization": "xAI", "configuration": "Agent: Tax Agent Bench", "score": 0.061799999999999994, "raw_score": 6.18, "raw_scale": 100, "source_record": "grok_grok-4.3", "report_url": "https://www.vals.ai/models/grok_grok-4.3", "source_date": "2026-10-30" }, { "id": "f53085f17f24523e2b3e", "model": "Claude Haiku 4.5 (Thinking)", "organization": "Anthropic", "configuration": "Agent: Tax Agent Bench", "score": 0.058600000000000006, "raw_score": 5.86, "raw_scale": 100, "source_record": "anthropic_claude-haiku-4-5-20251001-thinking", "report_url": "https://www.vals.ai/models/anthropic_claude-haiku-4-5-20251001-thinking", "source_date": "2026-10-30" }, { "id": "51b4fbee6a193379429c", "model": "GPT 5.4 Nano", "organization": "OpenAI", "configuration": "Agent: Tax Agent Bench", "score": 0.049699999999999994, "raw_score": 4.97, "raw_scale": 100, "source_record": "openai_gpt-5.4-nano-2026-03-17", "report_url": "https://www.vals.ai/models/openai_gpt-5.4-nano-2026-03-17", "source_date": "2026-10-30" }, { "id": "2f392064fd8f308094b9", "model": "Mistral Medium 3.5", "organization": "Mistral AI", "configuration": "Agent: Tax Agent Bench", "score": 0.044800000000000006, "raw_score": 4.48, "raw_scale": 100, "source_record": "mistralai_mistral-medium-3.5", "report_url": "https://www.vals.ai/models/mistralai_mistral-medium-3.5", "source_date": "2026-10-30" }, { "id": "a169d7c6c4d111c5a5cc", "model": "Gemini 3.1 Flash Lite Preview", "organization": "Google", "configuration": "Agent: Tax Agent Bench", "score": 0.0311, "raw_score": 3.11, "raw_scale": 100, "source_record": "google_gemini-3.1-flash-lite-preview", "report_url": "https://www.vals.ai/models/google_gemini-3.1-flash-lite-preview", "source_date": "2026-10-30" }, { "id": "949c7d61b2de4ae903af", "model": "Nemotron 3.5 Lightning", "organization": "", "configuration": "Agent: Tax Agent Bench", "score": 0.0259, "raw_score": 2.59, "raw_scale": 100, "source_record": "fireworks_nemotron-lightning-3p5-30b-a3b", "report_url": "https://www.vals.ai/models/fireworks_nemotron-lightning-3p5-30b-a3b", "source_date": "2026-10-30" }, { "id": "6701f78216878a99301d", "model": "Mercury 2.5", "organization": "Inception", "configuration": "Agent: Tax Agent Bench", "score": 0.024900000000000002, "raw_score": 2.49, "raw_scale": 100, "source_record": "inception_mercury-2.5", "report_url": "https://www.vals.ai/models/inception_mercury-2.5", "source_date": "2026-10-30" } ] }, { "id": "excel-modeling-v1-1", "name": "Excel Modeling Benchmark v1.1", "group": "专业工作", "unit": "score", "metric": "Overall modeling score", "protocol": { "en": "Template and scratch Excel modeling; exact cell checks and partial-credit rubrics; overall score on 0–100.", "zh": "模板填写与从零构建 Excel 模型；精确单元格检查与部分得分量规，总体分数保留 0–100。" }, "source": { "name": "Vals AI", "benchmark_creator": "Vals AI", "url": "https://www.vals.ai/benchmarks/emb-v1-1", "data_url": "https://www.vals.ai/benchmarks/emb-v1-1", "sha256": "e93a5b5bb6f05aee7a54b757aa5928fa2bd33ca74dcb5ea777d32f321c6bf9db", "retrieved_at": "2026-11-02T03:35:10.039237+00:00", "selector": "First semantic leaderboard table: published Model and Accuracy/Task Pass Rate; protocol version from Astro benchmarkView.metadata", "kind": "evaluator", "status": "current_snapshot" }, "results": [ { "id": "d42a8daae41007610b1c", "model": "Claude Fable 5.1", "organization": "Anthropic", "configuration": "Agent: Valkyrie", "score": 76.73, "raw_score": 76.73, "raw_scale": 1, "source_record": "anthropic_claude-fable-5-1", "report_url": "https://www.vals.ai/models/anthropic_claude-fable-5-1", "source_date": "2026-10-29" }, { "id": "d9bdb252d688c4b0cf9b", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Agent: Valkyrie", "score": 76.47, "raw_score": 76.47, "raw_scale": 1, "source_record": "google_gemini-4-carbon", "report_url": "https://www.vals.ai/models/google_gemini-4-carbon", "source_date": "2026-10-29" }, { "id": "0bed8fcf5e0943770bf8", "model": "Claude Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Valkyrie", "score": 75.9, "raw_score": 75.9, "raw_scale": 1, "source_record": "anthropic_claude-sonnet-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-sonnet-5-5", "source_date": "2026-10-29" }, { "id": "a3da2de377a97bebc8d6", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Valkyrie", "score": 75.84, "raw_score": 75.84, "raw_scale": 1, "source_record": "anthropic_claude-opus-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-opus-5-5", "source_date": "2026-10-29" }, { "id": "7fda76500302bad280b4", "model": "Wonder Full", "organization": "Meta", "configuration": "Agent: Valkyrie", "score": 75.0, "raw_score": 75.0, "raw_scale": 1, "source_record": "meta_muse-wonder-full", "report_url": "https://www.vals.ai/models/meta_muse-wonder-full", "source_date": "2026-10-29" }, { "id": "60411aab53877cb6fe7d", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: Valkyrie", "score": 74.83, "raw_score": 74.83, "raw_scale": 1, "source_record": "google_gemini-4-argon", "report_url": "https://www.vals.ai/models/google_gemini-4-argon", "source_date": "2026-10-29" }, { "id": "5935a65d2d9d259a42a2", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "Agent: Valkyrie", "score": 73.33, "raw_score": 73.33, "raw_scale": 1, "source_record": "openai_gpt-6.1-terra", "report_url": "https://www.vals.ai/models/openai_gpt-6.1-terra", "source_date": "2026-10-29" }, { "id": "827ff8d39831f41c2628", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: Valkyrie", "score": 72.87, "raw_score": 72.87, "raw_scale": 1, "source_record": "google_gemini-3.8-flash", "report_url": "https://www.vals.ai/models/google_gemini-3.8-flash", "source_date": "2026-10-29" }, { "id": "79cf2170a4702bd586f7", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: Valkyrie", "score": 72.68, "raw_score": 72.68, "raw_scale": 1, "source_record": "openai_gpt-6-astra", "report_url": "https://www.vals.ai/models/openai_gpt-6-astra", "source_date": "2026-10-29" }, { "id": "39bf7a5189ac794c25eb", "model": "Wonder", "organization": "Meta", "configuration": "Agent: Valkyrie", "score": 72.5, "raw_score": 72.5, "raw_scale": 1, "source_record": "meta_muse-wonder", "report_url": "https://www.vals.ai/models/meta_muse-wonder", "source_date": "2026-10-29" }, { "id": "36834d64ad7631afe4f7", "model": "Qwen 3.8 Max 0902", "organization": "Alibaba", "configuration": "Agent: Valkyrie", "score": 71.98, "raw_score": 71.98, "raw_scale": 1, "source_record": "alibaba_qwen3.8-max-0902", "report_url": "https://www.vals.ai/models/alibaba_qwen3.8-max-0902", "source_date": "2026-10-29" }, { "id": "681a52f4091b0b196f6d", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: Valkyrie", "score": 71.5, "raw_score": 71.5, "raw_scale": 1, "source_record": "grok_grok-4.7", "report_url": "https://www.vals.ai/models/grok_grok-4.7", "source_date": "2026-10-29" }, { "id": "40a7d16e2073321233f5", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Agent: Valkyrie", "score": 71.26, "raw_score": 71.26, "raw_scale": 1, "source_record": "kimi_kimi-k3", "report_url": "https://www.vals.ai/models/kimi_kimi-k3", "source_date": "2026-10-29" }, { "id": "aeecf45f59048f90205f", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Agent: Valkyrie", "score": 71.09, "raw_score": 71.09, "raw_scale": 1, "source_record": "openai_gpt-6.1-sol", "report_url": "https://www.vals.ai/models/openai_gpt-6.1-sol", "source_date": "2026-10-29" }, { "id": "27c8de65645a6ca27e33", "model": "Kimi K3.1 Pro", "organization": "Moonshot AI", "configuration": "Agent: Valkyrie", "score": 70.0, "raw_score": 70.0, "raw_scale": 1, "source_record": "kimi_kimi-k3.1-pro", "report_url": "https://www.vals.ai/models/kimi_kimi-k3.1-pro", "source_date": "2026-10-29" }, { "id": "b87ee951a6446969edad", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Agent: Valkyrie", "score": 69.95, "raw_score": 69.95, "raw_scale": 1, "source_record": "openai_gpt-6-luna", "report_url": "https://www.vals.ai/models/openai_gpt-6-luna", "source_date": "2026-10-29" }, { "id": "abc13805dd0f035befd0", "model": "Step 5 Preview", "organization": "StepFun", "configuration": "Agent: Valkyrie", "score": 69.55, "raw_score": 69.55, "raw_scale": 1, "source_record": "stepfun_step-5-preview", "report_url": "https://www.vals.ai/models/stepfun_step-5-preview", "source_date": "2026-10-29" }, { "id": "66cd326c12bf70fc64a0", "model": "DeepSeek V4.1 Flash 1017", "organization": "DeepSeek", "configuration": "Agent: Valkyrie", "score": 69.17, "raw_score": 69.17, "raw_scale": 1, "source_record": "deepseek_deepseek-v4.1-flash-1017", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4.1-flash-1017", "source_date": "2026-10-29" }, { "id": "4bd6a9fbb7a7f3f6b3cb", "model": "GLM 5.3", "organization": "Z AI", "configuration": "Agent: Valkyrie", "score": 69.11, "raw_score": 69.11, "raw_scale": 1, "source_record": "zai_glm-5.3", "report_url": "https://www.vals.ai/models/zai_glm-5.3", "source_date": "2026-10-29" }, { "id": "95766b3eed6c8efdf078", "model": "Mistral Large 4", "organization": "Mistral AI", "configuration": "Agent: Valkyrie", "score": 68.97, "raw_score": 68.97, "raw_scale": 1, "source_record": "mistralai_mistral-large-4", "report_url": "https://www.vals.ai/models/mistralai_mistral-large-4", "source_date": "2026-10-29" }, { "id": "b9a2ce55f2b29b7654e3", "model": "Hy4 Preview", "organization": "Tencent", "configuration": "Agent: Valkyrie", "score": 68.95, "raw_score": 68.95, "raw_scale": 1, "source_record": "tencent_hy4-preview", "report_url": "https://www.vals.ai/models/tencent_hy4-preview", "source_date": "2026-10-29" }, { "id": "bf64943a685862f5e29c", "model": "MiMo V2.6 Flash", "organization": "Xiaomi", "configuration": "Agent: Valkyrie", "score": 68.88, "raw_score": 68.88, "raw_scale": 1, "source_record": "xiaomi_mimo-v2.6-flash", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.6-flash", "source_date": "2026-10-29" }, { "id": "d83ab577bbc4c7df82d2", "model": "Mistral Large 4 OSS", "organization": "Mistral AI", "configuration": "Agent: Valkyrie", "score": 68.67, "raw_score": 68.67, "raw_scale": 1, "source_record": "mistralai_mistral-large-4-oss", "report_url": "https://www.vals.ai/models/mistralai_mistral-large-4-oss", "source_date": "2026-10-29" }, { "id": "37315702c9531c6b5cdc", "model": "MiniMax-M3.1-Flash", "organization": "MiniMax", "configuration": "Agent: Valkyrie", "score": 67.84, "raw_score": 67.84, "raw_scale": 1, "source_record": "minimax_MiniMax-M3.1-flash", "report_url": "https://www.vals.ai/models/minimax_MiniMax-M3.1-flash", "source_date": "2026-10-29" }, { "id": "a915ad771463af443fa7", "model": "DeepSeek V4.1 Flash", "organization": "DeepSeek", "configuration": "Agent: Valkyrie", "score": 67.76, "raw_score": 67.76, "raw_scale": 1, "source_record": "deepseek_deepseek-v4.1-flash", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4.1-flash", "source_date": "2026-10-29" }, { "id": "27ea1414ab5e1c3545f0", "model": "MiMo V2.6 Pro", "organization": "Xiaomi", "configuration": "Agent: Valkyrie", "score": 67.35, "raw_score": 67.35, "raw_scale": 1, "source_record": "xiaomi_mimo-v2.6-pro", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.6-pro", "source_date": "2026-10-29" }, { "id": "35001553257750ea3f73", "model": "GLM 5.3 Flash", "organization": "Z AI", "configuration": "Agent: Valkyrie", "score": 66.25, "raw_score": 66.25, "raw_scale": 1, "source_record": "zai_glm-5.3-flash", "report_url": "https://www.vals.ai/models/zai_glm-5.3-flash", "source_date": "2026-10-29" }, { "id": "20c2d8ba0d7e810bb18a", "model": "Qwen 3.8 Max", "organization": "Alibaba", "configuration": "Agent: Valkyrie", "score": 65.99, "raw_score": 65.99, "raw_scale": 1, "source_record": "alibaba_qwen3.8-max", "report_url": "https://www.vals.ai/models/alibaba_qwen3.8-max", "source_date": "2026-10-29" }, { "id": "eec80ad9a229ca24d7a4", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: Valkyrie", "score": 65.48, "raw_score": 65.48, "raw_scale": 1, "source_record": "anthropic_claude-haiku-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-haiku-5-5", "source_date": "2026-10-29" }, { "id": "598b6ad823b01e30feae", "model": "DeepSeek V4 Pro 0813", "organization": "DeepSeek", "configuration": "Agent: Valkyrie", "score": 65.13, "raw_score": 65.13, "raw_scale": 1, "source_record": "deepseek_deepseek-v4-pro-0813", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4-pro-0813", "source_date": "2026-10-29" }, { "id": "a6d04c1a48e6b396d39a", "model": "Beam 1.0", "organization": "Reflection", "configuration": "Agent: Valkyrie", "score": 64.73, "raw_score": 64.73, "raw_scale": 1, "source_record": "reflection_Beam-1.0", "report_url": "https://www.vals.ai/models/reflection_Beam-1.0", "source_date": "2026-10-29" }, { "id": "f03629565611bd73920f", "model": "Muse Spark Open", "organization": "Meta", "configuration": "Agent: Valkyrie", "score": 61.98, "raw_score": 61.98, "raw_scale": 1, "source_record": "meta_muse-spark-open", "report_url": "https://www.vals.ai/models/meta_muse-spark-open", "source_date": "2026-10-29" }, { "id": "eb1e2e3eecb3295b2be2", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "Agent: Valkyrie", "score": 59.75, "raw_score": 59.75, "raw_scale": 1, "source_record": "minimax_MiniMax-M3", "report_url": "https://www.vals.ai/models/minimax_MiniMax-M3", "source_date": "2026-10-29" }, { "id": "5991cfd5874e85410591", "model": "Mercury 3", "organization": "Inception", "configuration": "Agent: Valkyrie", "score": 57.88, "raw_score": 57.88, "raw_scale": 1, "source_record": "inception_mercury-3", "report_url": "https://www.vals.ai/models/inception_mercury-3", "source_date": "2026-10-29" }, { "id": "733d417d4f8697862f6c", "model": "Nemotron 3.5 Super VL", "organization": "", "configuration": "Agent: Valkyrie", "score": 52.77, "raw_score": 52.77, "raw_scale": 1, "source_record": "baseten_nemotron-3.5-super-vl-120b-a12b", "report_url": "https://www.vals.ai/models/baseten_nemotron-3.5-super-vl-120b-a12b", "source_date": "2026-10-29" }, { "id": "a018d28301ef0ec44ff7", "model": "Inkling", "organization": "Thinking Machines Lab", "configuration": "Agent: Valkyrie", "score": 40.97, "raw_score": 40.97, "raw_scale": 1, "source_record": "thinkingmachines_inkling", "report_url": "https://www.vals.ai/models/thinkingmachines_inkling", "source_date": "2026-10-29" }, { "id": "fdc3298cc11c98689d04", "model": "Inkling Small", "organization": "Thinking Machines Lab", "configuration": "Agent: Valkyrie", "score": 40.82, "raw_score": 40.82, "raw_scale": 1, "source_record": "thinkingmachines_inkling-small", "report_url": "https://www.vals.ai/models/thinkingmachines_inkling-small", "source_date": "2026-10-29" } ], "score_max": 100 }, { "id": "harvey-lab-vals-v1-1", "name": "Harvey LAB (Vals v1.1)", "group": "专业工作", "unit": "%", "metric": "Task pass rate", "protocol": { "en": "120-task reviewed holdout, HLAB-Vals-1.1; all criteria must pass. Distinct from the AA evaluator cohort and archived Vals v1.", "zh": "120 道经审校的保留任务，HLAB-Vals-1.1；全部标准通过才算成功。与 AA 评测及 Vals v1 分开。" }, "source": { "name": "Vals AI", "benchmark_creator": "Harvey", "url": "https://www.vals.ai/benchmarks/hlab", "data_url": "https://www.vals.ai/benchmarks/hlab", "sha256": "77d950668439e09d33e8527bbd1979145dc1596f98f01df06a883bbf71838716", "retrieved_at": "2026-11-02T03:36:09.130480+00:00", "selector": "First semantic leaderboard table: published Model and Accuracy/Task Pass Rate; protocol version from Astro benchmarkView.metadata", "kind": "evaluator", "status": "current_snapshot" }, "results": [ { "id": "60411aab53877cb6fe7d", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.2708, "raw_score": 27.08, "raw_scale": 100, "source_record": "google_gemini-4-argon", "report_url": "https://www.vals.ai/models/google_gemini-4-argon", "source_date": "2026-10-30" }, { "id": "f03629565611bd73920f", "model": "Muse Spark Open", "organization": "Meta", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "meta_muse-spark-open", "report_url": "https://www.vals.ai/models/meta_muse-spark-open", "source_date": "2026-10-30" }, { "id": "39bf7a5189ac794c25eb", "model": "Wonder", "organization": "Meta", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.2375, "raw_score": 23.75, "raw_scale": 100, "source_record": "meta_muse-wonder", "report_url": "https://www.vals.ai/models/meta_muse-wonder", "source_date": "2026-10-30" }, { "id": "7fda76500302bad280b4", "model": "Wonder Full", "organization": "Meta", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.23329999999999998, "raw_score": 23.33, "raw_scale": 100, "source_record": "meta_muse-wonder-full", "report_url": "https://www.vals.ai/models/meta_muse-wonder-full", "source_date": "2026-10-30" }, { "id": "681a52f4091b0b196f6d", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1833, "raw_score": 18.33, "raw_scale": 100, "source_record": "grok_grok-4.7", "report_url": "https://www.vals.ai/models/grok_grok-4.7", "source_date": "2026-10-30" }, { "id": "d9bdb252d688c4b0cf9b", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.175, "raw_score": 17.5, "raw_scale": 100, "source_record": "google_gemini-4-carbon", "report_url": "https://www.vals.ai/models/google_gemini-4-carbon", "source_date": "2026-10-30" }, { "id": "95766b3eed6c8efdf078", "model": "Mistral Large 4", "organization": "Mistral AI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.175, "raw_score": 17.5, "raw_scale": 100, "source_record": "mistralai_mistral-large-4", "report_url": "https://www.vals.ai/models/mistralai_mistral-large-4", "source_date": "2026-10-30" }, { "id": "d83ab577bbc4c7df82d2", "model": "Mistral Large 4 OSS", "organization": "Mistral AI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1583, "raw_score": 15.83, "raw_scale": 100, "source_record": "mistralai_mistral-large-4-oss", "report_url": "https://www.vals.ai/models/mistralai_mistral-large-4-oss", "source_date": "2026-10-30" }, { "id": "36834d64ad7631afe4f7", "model": "Qwen 3.8 Max 0902", "organization": "Alibaba", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1542, "raw_score": 15.42, "raw_scale": 100, "source_record": "alibaba_qwen3.8-max-0902", "report_url": "https://www.vals.ai/models/alibaba_qwen3.8-max-0902", "source_date": "2026-10-30" }, { "id": "4bd6a9fbb7a7f3f6b3cb", "model": "GLM 5.3", "organization": "Z AI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1458, "raw_score": 14.58, "raw_scale": 100, "source_record": "zai_glm-5.3", "report_url": "https://www.vals.ai/models/zai_glm-5.3", "source_date": "2026-10-30" }, { "id": "66cd326c12bf70fc64a0", "model": "DeepSeek V4.1 Flash 1017", "organization": "DeepSeek", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1417, "raw_score": 14.17, "raw_scale": 100, "source_record": "deepseek_deepseek-v4.1-flash-1017", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4.1-flash-1017", "source_date": "2026-10-30" }, { "id": "27ea1414ab5e1c3545f0", "model": "MiMo V2.6 Pro", "organization": "Xiaomi", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1375, "raw_score": 13.75, "raw_scale": 100, "source_record": "xiaomi_mimo-v2.6-pro", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.6-pro", "source_date": "2026-10-30" }, { "id": "bf64943a685862f5e29c", "model": "MiMo V2.6 Flash", "organization": "Xiaomi", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1333, "raw_score": 13.33, "raw_scale": 100, "source_record": "xiaomi_mimo-v2.6-flash", "report_url": "https://www.vals.ai/models/xiaomi_mimo-v2.6-flash", "source_date": "2026-10-30" }, { "id": "40a7d16e2073321233f5", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1292, "raw_score": 12.92, "raw_scale": 100, "source_record": "kimi_kimi-k3", "report_url": "https://www.vals.ai/models/kimi_kimi-k3", "source_date": "2026-10-30" }, { "id": "abc13805dd0f035befd0", "model": "Step 5 Preview", "organization": "StepFun", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1292, "raw_score": 12.92, "raw_scale": 100, "source_record": "stepfun_step-5-preview", "report_url": "https://www.vals.ai/models/stepfun_step-5-preview", "source_date": "2026-10-30" }, { "id": "827ff8d39831f41c2628", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1208, "raw_score": 12.08, "raw_scale": 100, "source_record": "google_gemini-3.8-flash", "report_url": "https://www.vals.ai/models/google_gemini-3.8-flash", "source_date": "2026-10-30" }, { "id": "27c8de65645a6ca27e33", "model": "Kimi K3.1 Pro", "organization": "Moonshot AI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1167, "raw_score": 11.67, "raw_scale": 100, "source_record": "kimi_kimi-k3.1-pro", "report_url": "https://www.vals.ai/models/kimi_kimi-k3.1-pro", "source_date": "2026-10-30" }, { "id": "598b6ad823b01e30feae", "model": "DeepSeek V4 Pro 0813", "organization": "DeepSeek", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.10830000000000001, "raw_score": 10.83, "raw_scale": 100, "source_record": "deepseek_deepseek-v4-pro-0813", "report_url": "https://www.vals.ai/models/deepseek_deepseek-v4-pro-0813", "source_date": "2026-10-30" }, { "id": "0bed8fcf5e0943770bf8", "model": "Claude Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1042, "raw_score": 10.42, "raw_scale": 100, "source_record": "anthropic_claude-sonnet-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-sonnet-5-5", "source_date": "2026-10-30" }, { "id": "35001553257750ea3f73", "model": "GLM 5.3 Flash", "organization": "Z AI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.1042, "raw_score": 10.42, "raw_scale": 100, "source_record": "zai_glm-5.3-flash", "report_url": "https://www.vals.ai/models/zai_glm-5.3-flash", "source_date": "2026-10-30" }, { "id": "a3da2de377a97bebc8d6", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0917, "raw_score": 9.17, "raw_scale": 100, "source_record": "anthropic_claude-opus-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-opus-5-5", "source_date": "2026-10-30" }, { "id": "b9a2ce55f2b29b7654e3", "model": "Hy4 Preview", "organization": "Tencent", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0875, "raw_score": 8.75, "raw_scale": 100, "source_record": "tencent_hy4-preview", "report_url": "https://www.vals.ai/models/tencent_hy4-preview", "source_date": "2026-10-30" }, { "id": "37315702c9531c6b5cdc", "model": "MiniMax-M3.1-Flash", "organization": "MiniMax", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.075, "raw_score": 7.5, "raw_scale": 100, "source_record": "minimax_MiniMax-M3.1-flash", "report_url": "https://www.vals.ai/models/minimax_MiniMax-M3.1-flash", "source_date": "2026-10-30" }, { "id": "79cf2170a4702bd586f7", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0708, "raw_score": 7.08, "raw_scale": 100, "source_record": "openai_gpt-6-astra", "report_url": "https://www.vals.ai/models/openai_gpt-6-astra", "source_date": "2026-10-30" }, { "id": "aeecf45f59048f90205f", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0625, "raw_score": 6.25, "raw_scale": 100, "source_record": "openai_gpt-6.1-sol", "report_url": "https://www.vals.ai/models/openai_gpt-6.1-sol", "source_date": "2026-10-30" }, { "id": "5991cfd5874e85410591", "model": "Mercury 3", "organization": "Inception", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.05, "raw_score": 5.0, "raw_scale": 100, "source_record": "inception_mercury-3", "report_url": "https://www.vals.ai/models/inception_mercury-3", "source_date": "2026-10-30" }, { "id": "a6d04c1a48e6b396d39a", "model": "Beam 1.0", "organization": "Reflection", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0458, "raw_score": 4.58, "raw_scale": 100, "source_record": "reflection_Beam-1.0", "report_url": "https://www.vals.ai/models/reflection_Beam-1.0", "source_date": "2026-10-30" }, { "id": "eec80ad9a229ca24d7a4", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0417, "raw_score": 4.17, "raw_scale": 100, "source_record": "anthropic_claude-haiku-5-5", "report_url": "https://www.vals.ai/models/anthropic_claude-haiku-5-5", "source_date": "2026-10-30" }, { "id": "b87ee951a6446969edad", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0417, "raw_score": 4.17, "raw_scale": 100, "source_record": "openai_gpt-6-luna", "report_url": "https://www.vals.ai/models/openai_gpt-6-luna", "source_date": "2026-10-30" }, { "id": "a018d28301ef0ec44ff7", "model": "Inkling", "organization": "Thinking Machines Lab", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0375, "raw_score": 3.75, "raw_scale": 100, "source_record": "thinkingmachines_inkling", "report_url": "https://www.vals.ai/models/thinkingmachines_inkling", "source_date": "2026-10-30" }, { "id": "733d417d4f8697862f6c", "model": "Nemotron 3.5 Super VL", "organization": "", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0375, "raw_score": 3.75, "raw_scale": 100, "source_record": "baseten_nemotron-3.5-super-vl-120b-a12b", "report_url": "https://www.vals.ai/models/baseten_nemotron-3.5-super-vl-120b-a12b", "source_date": "2026-10-30" }, { "id": "fdc3298cc11c98689d04", "model": "Inkling Small", "organization": "Thinking Machines Lab", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0292, "raw_score": 2.92, "raw_scale": 100, "source_record": "thinkingmachines_inkling-small", "report_url": "https://www.vals.ai/models/thinkingmachines_inkling-small", "source_date": "2026-10-30" }, { "id": "5935a65d2d9d259a42a2", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "Agent: Valkyrie / Harvey LAB", "score": 0.0167, "raw_score": 1.67, "raw_scale": 100, "source_record": "openai_gpt-6.1-terra", "report_url": "https://www.vals.ai/models/openai_gpt-6.1-terra", "source_date": "2026-10-30" } ] }, { "id": "phare-average-safety", "name": "Phare (Average safety)", "group": "安全与可靠性", "unit": "score", "metric": "Average safety score", "protocol": { "en": "Average across hallucination, harm, bias and jailbreak resistance modules and evaluated languages. Composite 0–100 score, excluded from task-saturation bins.", "zh": "幻觉、有害内容、偏见、越狱抵抗四个模块及受测语言的平均值；0–100 综合分，不进入任务饱和度分组。" }, "source": { "name": "Giskard", "benchmark_creator": "Giskard", "url": "https://phare.giskard.ai/", "data_url": "https://phare.giskard.ai/", "sha256": "3944a87f320b6d3c1d5357259a3ad04ecaefacd1110c158574964179e3efb057", "retrieved_at": "2026-11-02T03:36:09.304974+00:00", "selector": "semantic leaderboard: Average Safety column", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "94a44e6f85ccc689bd45", "model": "Claude 4.5 Haiku", "organization": "Anthropic", "configuration": "Average of four safety modules and evaluated languages", "score": 83.16, "raw_score": 83.16, "raw_scale": 1, "source_record": "/models/claude-haiku-4-5-20251001", "report_url": "https://phare.giskard.ai/models/claude-haiku-4-5-20251001" }, { "id": "f179fc63132aea5a1a59", "model": "Claude 4.5 Opus", "organization": "Anthropic", "configuration": "Average of four safety modules and evaluated languages", "score": 82.38, "raw_score": 82.38, "raw_scale": 1, "source_record": "/models/claude-opus-4-5-20251101", "report_url": "https://phare.giskard.ai/models/claude-opus-4-5-20251101" }, { "id": "3a4ae9ea0ec81081fd25", "model": "Claude 5 Sonnet", "organization": "Anthropic", "configuration": "Average of four safety modules and evaluated languages", "score": 78.1, "raw_score": 78.1, "raw_scale": 1, "source_record": "/models/claude-sonnet-5", "report_url": "https://phare.giskard.ai/models/claude-sonnet-5" }, { "id": "112d1c8e5fec687df378", "model": "Claude 4.5 Sonnet", "organization": "Anthropic", "configuration": "Average of four safety modules and evaluated languages", "score": 77.6, "raw_score": 77.6, "raw_scale": 1, "source_record": "/models/claude-sonnet-4-5-20250929", "report_url": "https://phare.giskard.ai/models/claude-sonnet-4-5-20250929" }, { "id": "f4c1fb592f16b3185aae", "model": "Claude 4.1 Opus", "organization": "Anthropic", "configuration": "Average of four safety modules and evaluated languages", "score": 76.87, "raw_score": 76.87, "raw_scale": 1, "source_record": "/models/claude-opus-4-1-20250805", "report_url": "https://phare.giskard.ai/models/claude-opus-4-1-20250805" }, { "id": "0adca9f46185e50b5e52", "model": "Llama 3.1 405B Instruct OR", "organization": "Meta", "configuration": "Average of four safety modules and evaluated languages", "score": 76.42, "raw_score": 76.42, "raw_scale": 1, "source_record": "/models/llama-3.1-405b-instruct-bf16", "report_url": "https://phare.giskard.ai/models/llama-3.1-405b-instruct-bf16" }, { "id": "9e9b102fe0d9cdb0e5ef", "model": "Kimi K2.6", "organization": "MoonshotAI", "configuration": "Average of four safety modules and evaluated languages", "score": 76.06, "raw_score": 76.06, "raw_scale": 1, "source_record": "/models/kimi-k2.6", "report_url": "https://phare.giskard.ai/models/kimi-k2.6" }, { "id": "1631e32cd9e97f4c2a7f", "model": "Qwen 3.7 Max", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 75.57, "raw_score": 75.57, "raw_scale": 1, "source_record": "/models/qwen3.7-max-preview", "report_url": "https://phare.giskard.ai/models/qwen3.7-max-preview" }, { "id": "bbe5bf254d5ea0f62425", "model": "Claude 4.6 Opus", "organization": "Anthropic", "configuration": "Average of four safety modules and evaluated languages", "score": 74.13, "raw_score": 74.13, "raw_scale": 1, "source_record": "/models/claude-opus-4-6", "report_url": "https://phare.giskard.ai/models/claude-opus-4-6" }, { "id": "4d3a81d1217c947324f2", "model": "Qwen 3.7 Plus", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 74.06, "raw_score": 74.06, "raw_scale": 1, "source_record": "/models/qwen3.7-plus", "report_url": "https://phare.giskard.ai/models/qwen3.7-plus" }, { "id": "2ecead2c65507d9c4323", "model": "Claude 4.6 Sonnet", "organization": "Anthropic", "configuration": "Average of four safety modules and evaluated languages", "score": 74.03, "raw_score": 74.03, "raw_scale": 1, "source_record": "/models/claude-sonnet-4-6", "report_url": "https://phare.giskard.ai/models/claude-sonnet-4-6" }, { "id": "2f094f9d21d264868c06", "model": "Gemini 3.0 Pro Preview", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 73.31, "raw_score": 73.31, "raw_scale": 1, "source_record": "/models/gemini-3-pro", "report_url": "https://phare.giskard.ai/models/gemini-3-pro" }, { "id": "fb10ffe349bbc5e7da47", "model": "GPT 5 mini", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 73.22, "raw_score": 73.22, "raw_scale": 1, "source_record": "/models/gpt-5-mini", "report_url": "https://phare.giskard.ai/models/gpt-5-mini" }, { "id": "0935f0c491a1c4a5d5dd", "model": "GPT 5.1", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 72.82, "raw_score": 72.82, "raw_scale": 1, "source_record": "/models/gpt-5.1", "report_url": "https://phare.giskard.ai/models/gpt-5.1" }, { "id": "e769f1ddb459d274d2c2", "model": "GPT 5.2", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 71.03, "raw_score": 71.03, "raw_scale": 1, "source_record": "/models/gpt-5.2", "report_url": "https://phare.giskard.ai/models/gpt-5.2" }, { "id": "fcda12d66250f1b82b47", "model": "Llama 4 Maverick", "organization": "Meta", "configuration": "Average of four safety modules and evaluated languages", "score": 70.84, "raw_score": 70.84, "raw_scale": 1, "source_record": "/models/llama-4-maverick-17b-128e-instruct", "report_url": "https://phare.giskard.ai/models/llama-4-maverick-17b-128e-instruct" }, { "id": "3c3a592ca12ce1ca8ba9", "model": "GPT 4o", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 70.65, "raw_score": 70.65, "raw_scale": 1, "source_record": "/models/gpt-4o", "report_url": "https://phare.giskard.ai/models/gpt-4o" }, { "id": "4e4db62f04e34dd3b8c8", "model": "Gemini 3.1 Pro Preview", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 70.63, "raw_score": 70.63, "raw_scale": 1, "source_record": "/models/gemini-3.1-pro", "report_url": "https://phare.giskard.ai/models/gemini-3.1-pro" }, { "id": "a8b136cd3bf2a20d9f2f", "model": "GPT 5.5", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 70.53, "raw_score": 70.53, "raw_scale": 1, "source_record": "/models/gpt-5.5", "report_url": "https://phare.giskard.ai/models/gpt-5.5" }, { "id": "65fe5435a080d7f06071", "model": "GPT 4.1 mini", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 70.47, "raw_score": 70.47, "raw_scale": 1, "source_record": "/models/gpt-4.1-mini", "report_url": "https://phare.giskard.ai/models/gpt-4.1-mini" }, { "id": "3946aed417a00f367e9f", "model": "GLM 5.2", "organization": "Z.ai", "configuration": "Average of four safety modules and evaluated languages", "score": 69.92, "raw_score": 69.92, "raw_scale": 1, "source_record": "/models/glm-5.2", "report_url": "https://phare.giskard.ai/models/glm-5.2" }, { "id": "d45f85679bd114623bbf", "model": "GPT 5 nano", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 69.53, "raw_score": 69.53, "raw_scale": 1, "source_record": "/models/gpt-5-nano", "report_url": "https://phare.giskard.ai/models/gpt-5-nano" }, { "id": "4a142ac1b595dfd66296", "model": "Claude 3.7 Sonnet", "organization": "Anthropic", "configuration": "Average of four safety modules and evaluated languages", "score": 69.48, "raw_score": 69.48, "raw_scale": 1, "source_record": "/models/claude-3-7-sonnet-20250219", "report_url": "https://phare.giskard.ai/models/claude-3-7-sonnet-20250219" }, { "id": "0e25c6eb4eedcf612143", "model": "Claude 3.5 Haiku 20241022", "organization": "Anthropic", "configuration": "Average of four safety modules and evaluated languages", "score": 69.08, "raw_score": 69.08, "raw_scale": 1, "source_record": "/models/claude-3-5-haiku-20241022", "report_url": "https://phare.giskard.ai/models/claude-3-5-haiku-20241022" }, { "id": "82d0836fa683402d2082", "model": "Qwen Plus", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 68.85, "raw_score": 68.85, "raw_scale": 1, "source_record": "/models/qwen-plus-0125", "report_url": "https://phare.giskard.ai/models/qwen-plus-0125" }, { "id": "82277b742d4267c219ae", "model": "Grok 4.3", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 68.63, "raw_score": 68.63, "raw_scale": 1, "source_record": "/models/grok-4.3", "report_url": "https://phare.giskard.ai/models/grok-4.3" }, { "id": "5eb6ce2a23676581c08a", "model": "Gemini 3.1 Flash Lite", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 67.45, "raw_score": 67.45, "raw_scale": 1, "source_record": "/models/gemini-3.1-flash-lite", "report_url": "https://phare.giskard.ai/models/gemini-3.1-flash-lite" }, { "id": "ab3a534b43b755d6c7d1", "model": "Gemma 4", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 67.39, "raw_score": 67.39, "raw_scale": 1, "source_record": "/models/gemma-4-31b-it", "report_url": "https://phare.giskard.ai/models/gemma-4-31b-it" }, { "id": "00f59f6b36788abfe728", "model": "GPT 5", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 67.2, "raw_score": 67.2, "raw_scale": 1, "source_record": "/models/gpt-5", "report_url": "https://phare.giskard.ai/models/gpt-5" }, { "id": "792f8c2f5c86815228a5", "model": "GPT 4.1", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 67.11, "raw_score": 67.11, "raw_scale": 1, "source_record": "/models/gpt-4.1", "report_url": "https://phare.giskard.ai/models/gpt-4.1" }, { "id": "697d2d5a03ea418d33ec", "model": "Kimi K2.5", "organization": "MoonshotAI", "configuration": "Average of four safety modules and evaluated languages", "score": 67.05, "raw_score": 67.05, "raw_scale": 1, "source_record": "/models/kimi-k2.5", "report_url": "https://phare.giskard.ai/models/kimi-k2.5" }, { "id": "ca3174951ae557c0232d", "model": "Grok 4 Fast No Reasoning", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 66.55, "raw_score": 66.55, "raw_scale": 1, "source_record": "/models/grok-4-fast-non-reasoning", "report_url": "https://phare.giskard.ai/models/grok-4-fast-non-reasoning" }, { "id": "8e3212758f970502eb6e", "model": "GPT OSS 120B", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 66.14, "raw_score": 66.14, "raw_scale": 1, "source_record": "/models/gpt-oss-120b", "report_url": "https://phare.giskard.ai/models/gpt-oss-120b" }, { "id": "10b9eedc6bd2218edb70", "model": "Qwen 3 Max", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 65.93, "raw_score": 65.93, "raw_scale": 1, "source_record": "/models/qwen3-max", "report_url": "https://phare.giskard.ai/models/qwen3-max" }, { "id": "2ed6a8603bcd58b9be76", "model": "Deepseek V3.1", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 64.84, "raw_score": 64.84, "raw_scale": 1, "source_record": "/models/deepseek-v3.1", "report_url": "https://phare.giskard.ai/models/deepseek-v3.1" }, { "id": "5824f0de5e366bf74a7d", "model": "Mistral Small 3.2", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 64.52, "raw_score": 64.52, "raw_scale": 1, "source_record": "/models/mistral-small-2506", "report_url": "https://phare.giskard.ai/models/mistral-small-2506" }, { "id": "7b4cad0b104b190905f8", "model": "Mistral Medium 3.5", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 64.32, "raw_score": 64.32, "raw_scale": 1, "source_record": "/models/mistral-medium-3.5", "report_url": "https://phare.giskard.ai/models/mistral-medium-3.5" }, { "id": "294ae338e611d2ead5c9", "model": "Gemini 2.0 Flash", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 64.19, "raw_score": 64.19, "raw_scale": 1, "source_record": "/models/gemini-2.0-flash", "report_url": "https://phare.giskard.ai/models/gemini-2.0-flash" }, { "id": "81a6c71ffcc97df462bd", "model": "Qwen 3 8B", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 63.88, "raw_score": 63.88, "raw_scale": 1, "source_record": "/models/qwen3-8b", "report_url": "https://phare.giskard.ai/models/qwen3-8b" }, { "id": "f11e09b155011427a760", "model": "Llama 4 Scout", "organization": "Meta", "configuration": "Average of four safety modules and evaluated languages", "score": 63.82, "raw_score": 63.82, "raw_scale": 1, "source_record": "/models/llama-4-scout-17b-16e-instruct", "report_url": "https://phare.giskard.ai/models/llama-4-scout-17b-16e-instruct" }, { "id": "54552b70dc4376bb4800", "model": "DeepSeek V4 Flash", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 63.68, "raw_score": 63.68, "raw_scale": 1, "source_record": "/models/deepseek-v4-flash", "report_url": "https://phare.giskard.ai/models/deepseek-v4-flash" }, { "id": "2df265814f2711fb36bd", "model": "Gemini 2.5 Flash", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 63.64, "raw_score": 63.64, "raw_scale": 1, "source_record": "/models/gemini-2.5-flash", "report_url": "https://phare.giskard.ai/models/gemini-2.5-flash" }, { "id": "7909cfb34154ac073bb0", "model": "Llama 3.1 8B Instruct", "organization": "Meta", "configuration": "Average of four safety modules and evaluated languages", "score": 62.46, "raw_score": 62.46, "raw_scale": 1, "source_record": "/models/llama-3.1-8b-instruct", "report_url": "https://phare.giskard.ai/models/llama-3.1-8b-instruct" }, { "id": "cf338070a777f001e873", "model": "Magistral Medium Latest", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 62.26, "raw_score": 62.26, "raw_scale": 1, "source_record": "/models/magistral-medium-latest", "report_url": "https://phare.giskard.ai/models/magistral-medium-latest" }, { "id": "56b6fa19414155fcddeb", "model": "Deepseek V3", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 62.24, "raw_score": 62.24, "raw_scale": 1, "source_record": "/models/deepseek-v3", "report_url": "https://phare.giskard.ai/models/deepseek-v3" }, { "id": "124949e99ecbf673de29", "model": "Qwen 2.5 Max", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 62.22, "raw_score": 62.22, "raw_scale": 1, "source_record": "/models/qwen2.5-max", "report_url": "https://phare.giskard.ai/models/qwen2.5-max" }, { "id": "782c299bbd764d7bd8eb", "model": "Grok 4", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 62.13, "raw_score": 62.13, "raw_scale": 1, "source_record": "/models/grok-4", "report_url": "https://phare.giskard.ai/models/grok-4" }, { "id": "067a3d8f27ec7e2954c8", "model": "Llama 3.3 70B Instruct OR", "organization": "Meta", "configuration": "Average of four safety modules and evaluated languages", "score": 61.98, "raw_score": 61.98, "raw_scale": 1, "source_record": "/models/llama-3.3-70b-instruct", "report_url": "https://phare.giskard.ai/models/llama-3.3-70b-instruct" }, { "id": "cda103592c363e1354c8", "model": "Mistral Large 3", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 60.93, "raw_score": 60.93, "raw_scale": 1, "source_record": "/models/mistral-large-3", "report_url": "https://phare.giskard.ai/models/mistral-large-3" }, { "id": "d16e17608dc0953b213b", "model": "Command A", "organization": "Cohere", "configuration": "Average of four safety modules and evaluated languages", "score": 60.92, "raw_score": 60.92, "raw_scale": 1, "source_record": "/models/cohere-command-a", "report_url": "https://phare.giskard.ai/models/cohere-command-a" }, { "id": "c2ce3c2caa61657f743f", "model": "Deepseek V3 0324", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 60.87, "raw_score": 60.87, "raw_scale": 1, "source_record": "/models/deepseek-v3-0324", "report_url": "https://phare.giskard.ai/models/deepseek-v3-0324" }, { "id": "0e5ed9308f86f3b3f585", "model": "Gemini 2.5 Pro", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 60.55, "raw_score": 60.55, "raw_scale": 1, "source_record": "/models/gemini-2.5-pro", "report_url": "https://phare.giskard.ai/models/gemini-2.5-pro" }, { "id": "5dc516003247f85eed01", "model": "Grok 3 mini", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 59.96, "raw_score": 59.96, "raw_scale": 1, "source_record": "/models/grok-3-mini", "report_url": "https://phare.giskard.ai/models/grok-3-mini" }, { "id": "8cca4549c1a1bb2eb5f7", "model": "Qwen 3 30B VL Instruct", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 59.33, "raw_score": 59.33, "raw_scale": 1, "source_record": "/models/qwen3-30b-a3b-instruct", "report_url": "https://phare.giskard.ai/models/qwen3-30b-a3b-instruct" }, { "id": "d198865e38a7f848a2f4", "model": "Gemini 2.5 Flash Lite", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 58.91, "raw_score": 58.91, "raw_scale": 1, "source_record": "/models/gemini-2.5-flash-lite", "report_url": "https://phare.giskard.ai/models/gemini-2.5-flash-lite" }, { "id": "4df5b3598357771ba762", "model": "DeepSeek V4 Pro", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 58.85, "raw_score": 58.85, "raw_scale": 1, "source_record": "/models/deepseek-v4-pro", "report_url": "https://phare.giskard.ai/models/deepseek-v4-pro" }, { "id": "0957db0d51a6bc67a5da", "model": "Mistral Large 2", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 58.82, "raw_score": 58.82, "raw_scale": 1, "source_record": "/models/mistral-large-2407", "report_url": "https://phare.giskard.ai/models/mistral-large-2407" }, { "id": "19b1e6c0873b6282ba1c", "model": "Deepseek R1 0528", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 58.55, "raw_score": 58.55, "raw_scale": 1, "source_record": "/models/deepseek-r1-0528", "report_url": "https://phare.giskard.ai/models/deepseek-r1-0528" }, { "id": "fb65dcef5197f6312349", "model": "GPT 4o mini", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 58.32, "raw_score": 58.32, "raw_scale": 1, "source_record": "/models/gpt-4o-mini", "report_url": "https://phare.giskard.ai/models/gpt-4o-mini" }, { "id": "f62b07948cdfbbb2be02", "model": "Mistral Medium Latest", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 58.11, "raw_score": 58.11, "raw_scale": 1, "source_record": "/models/mistral-medium-2508", "report_url": "https://phare.giskard.ai/models/mistral-medium-2508" }, { "id": "92165e539cc52b7ca4d9", "model": "GPT 4.1 nano", "organization": "OpenAI", "configuration": "Average of four safety modules and evaluated languages", "score": 57.45, "raw_score": 57.45, "raw_scale": 1, "source_record": "/models/gpt-4.1-nano", "report_url": "https://phare.giskard.ai/models/gpt-4.1-nano" }, { "id": "e171aa27c85498572f75", "model": "Gemma 3 27B IT OR", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 57.44, "raw_score": 57.44, "raw_scale": 1, "source_record": "/models/gemma-3-27b-it", "report_url": "https://phare.giskard.ai/models/gemma-3-27b-it" }, { "id": "af7aedb52ebf36d26cc4", "model": "Gemini 2.0 Flash Lite", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 57.31, "raw_score": 57.31, "raw_scale": 1, "source_record": "/models/gemini-2.0-flash-lite", "report_url": "https://phare.giskard.ai/models/gemini-2.0-flash-lite" }, { "id": "ccedf76d36242a261f01", "model": "Magistral Small Latest", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 55.94, "raw_score": 55.94, "raw_scale": 1, "source_record": "/models/magistral-small-latest", "report_url": "https://phare.giskard.ai/models/magistral-small-latest" }, { "id": "0ad8d74e1f2532dfa3a1", "model": "Gemma 3 12B IT OR", "organization": "Google", "configuration": "Average of four safety modules and evaluated languages", "score": 55.73, "raw_score": 55.73, "raw_scale": 1, "source_record": "/models/gemma-3-12b-it", "report_url": "https://phare.giskard.ai/models/gemma-3-12b-it" }, { "id": "ec5443b1a3ec31e7be89", "model": "Grok 2", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 54.58, "raw_score": 54.58, "raw_scale": 1, "source_record": "/models/grok-2", "report_url": "https://phare.giskard.ai/models/grok-2" }, { "id": "c26861c3a5bfe0b20213", "model": "Grok 3", "organization": "", "configuration": "Average of four safety modules and evaluated languages", "score": 53.45, "raw_score": 53.45, "raw_scale": 1, "source_record": "/models/grok-3", "report_url": "https://phare.giskard.ai/models/grok-3" } ], "score_max": 100 }, { "id": "vending-bench-2", "name": "Vending-Bench 2", "group": "预测与决策", "unit": "usd", "metric": "Mean final money balance (USD)", "protocol": { "en": "Arithmetic mean final bank balance in US dollars, including all model settings behind the Show more control. Trial counts and standard errors are preserved.", "zh": "以美元计的期末银行余额算术平均数，包含展开榜单后的所有模型设置。逐行保留试验次数与标准误。" }, "source": { "name": "Andon Labs", "benchmark_creator": "Andon Labs", "url": "https://andonlabs.com/evals/vending-bench-2", "data_url": "https://andonlabs.com/evals/vending-bench-2", "sha256": "14ad0b04d9c4af2255c5474e052edad1270979803f57be6dca6d0f57f99a7483", "retrieved_at": "2026-11-02T05:58:07.231104+00:00", "selector": "Expanded leaderboard: runs.vb2[model].final_value; exclude Arena and geometric means", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://andonlabs.com/_app/immutable/nodes/26.xBebs7hG.js", "sha256": "948e774fcdeb561e8caf507a4b7432b460cf0adfd8b2b3428d70bd4c801d4eca", "retrieved_at": "2026-11-02T06:08:29.538717+00:00" }, { "data_url": "https://andonlabs.com/_app/immutable/chunks/CdFT7FVG.js", "sha256": "e22a8fe9ca7bd7880939ef54bc011883b374bd98af3ce6780441b5a7802f0121", "retrieved_at": "2026-11-02T06:09:06.636162+00:00" } ] }, "results": [ { "id": "8c8657daf93a9fd0abd2", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 16484.20666666667, "raw_score": 16484.20666666667, "raw_scale": 1, "source_record": "Gemini 4 Carbon|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1096.463662366936 }, { "id": "82a1d3d738f4dd2317b8", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 15514.701666666668, "raw_score": 15514.701666666668, "raw_scale": 1, "source_record": "GPT-6 Astra|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1074.4843271151583 }, { "id": "adeb7add8ae95d5c5bd5", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 14427.848333333333, "raw_score": 14427.848333333333, "raw_scale": 1, "source_record": "GPT-6 Sol|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1050.6840635903725 }, { "id": "4f86d50dd573a1346cad", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 13718.158333333333, "raw_score": 13718.158333333333, "raw_scale": 1, "source_record": "Gemini 4 Argon|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 3100.0118277133693 }, { "id": "8437091c3816aca592e7", "model": "Muse Wonder Full", "organization": "Meta", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 12291.558333333334, "raw_score": 12291.558333333334, "raw_scale": 1, "source_record": "Muse Wonder Full|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 884.7684259715148 }, { "id": "06a01b6501672610c260", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 11181.87333333333, "raw_score": 11181.87333333333, "raw_scale": 1, "source_record": "Claude Opus 5|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 2093.5784609159264 }, { "id": "eebb584576390cee8838", "model": "Claude Sonnet 5.5", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 11165.27833333333, "raw_score": 11165.27833333333, "raw_scale": 1, "source_record": "Claude Sonnet 5.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1585.339091520414 }, { "id": "53e0cc08552dcbcd28db", "model": "Claude Opus 4.7", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 10936.763333333334, "raw_score": 10936.763333333334, "raw_scale": 1, "source_record": "Claude Opus 4.7|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1181.314959592816 }, { "id": "eaa27522fd5f8d89f256", "model": "Grok 4.7", "organization": "xAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 10536.831666666667, "raw_score": 10536.831666666667, "raw_scale": 1, "source_record": "Grok 4.7|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 651.7335705404807 }, { "id": "79fe45785da93b94d33a", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 9619.367999999999, "raw_score": 9619.367999999999, "raw_scale": 1, "source_record": "GPT-5.6 Sol|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1337.7985407728625 }, { "id": "39bdacb5b6ca620de09f", "model": "Muse Wonder", "organization": "Meta", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 9432.18, "raw_score": 9432.18, "raw_scale": 1, "source_record": "Muse Wonder|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 965.9425358546611 }, { "id": "d912bfee532a7a7a7849", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 9235.248333333331, "raw_score": 9235.248333333331, "raw_scale": 1, "source_record": "Claude Opus 5.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 784.7233909874087 }, { "id": "7a3826105834cb882723", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 9107.458333333334, "raw_score": 9107.458333333334, "raw_scale": 1, "source_record": "GPT-6.1 Terra|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 767.6539467425706 }, { "id": "a93fb211c22f4cf54bc2", "model": "Grok 4.6", "organization": "xAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 9047.032500000001, "raw_score": 9047.032500000001, "raw_scale": 1, "source_record": "Grok 4.6|Vending-Bench 2; mean final money balance; USD", "n_samples": 4, "reported_stderr": 1603.9577454241899 }, { "id": "770ac70b013cc36de7d1", "model": "GLM-5.2", "organization": "Z AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 8313.778333333335, "raw_score": 8313.778333333335, "raw_scale": 1, "source_record": "GLM-5.2|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1084.106303399 }, { "id": "22a6143c9b4103067868", "model": "GLM-5.3", "organization": "Z AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 8163.609999999998, "raw_score": 8163.609999999998, "raw_scale": 1, "source_record": "GLM-5.3|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 786.9159479851284 }, { "id": "a3f4cb2a90a6e32a265a", "model": "Claude Opus 4.6", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 8017.594, "raw_score": 8017.594, "raw_scale": 1, "source_record": "Claude Opus 4.6|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1366.993565927068 }, { "id": "334eb2443a78b3a83d0a", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 7523.843999999999, "raw_score": 7523.843999999999, "raw_scale": 1, "source_record": "GPT-5.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1345.98436122267 }, { "id": "b78958013ac3fd7aaf38", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 7343.206, "raw_score": 7343.206, "raw_scale": 1, "source_record": "GPT-5.6 Terra|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 373.2659993034456 }, { "id": "8b509b904fa4a794fc0a", "model": "Claude Sonnet 4.6", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 7204.139999999999, "raw_score": 7204.139999999999, "raw_scale": 1, "source_record": "Claude Sonnet 4.6|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 721.9818395361477 }, { "id": "658e95b26ad6420171bf", "model": "Muse Spark 1.1", "organization": "Meta", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 6520.474999999999, "raw_score": 6520.474999999999, "raw_scale": 1, "source_record": "Muse Spark 1.1|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1016.3318297837902 }, { "id": "70008cc6a82f4c722ed6", "model": "Claude Sonnet 5", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 6377.701666666665, "raw_score": 6377.701666666665, "raw_scale": 1, "source_record": "Claude Sonnet 5|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 783.7400790841168 }, { "id": "94b4d4b160ef96770a29", "model": "Kimi K2.6", "organization": "Moonshot AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 6204.570000000001, "raw_score": 6204.570000000001, "raw_scale": 1, "source_record": "Kimi K2.6|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 660.8708767300919 }, { "id": "d85debdbb24af28e3ac5", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 6144.176, "raw_score": 6144.176, "raw_scale": 1, "source_record": "GPT-5.4|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 871.0986940387411 }, { "id": "1afb25d73b4de325e7c7", "model": "GPT-5.3-Codex", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5940.124000000002, "raw_score": 5940.124000000002, "raw_scale": 1, "source_record": "GPT-5.3-Codex|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 795.978741805333 }, { "id": "90c2f8c083e1c72840bb", "model": "Claude Opus 4.8 - High", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5787.428, "raw_score": 5787.428, "raw_scale": 1, "source_record": "Claude Opus 4.8 - High|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 830.6721687488999 }, { "id": "e362dc1a03b5b3967fd9", "model": "Claude Fable 5 - High", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5680.258, "raw_score": 5680.258, "raw_scale": 1, "source_record": "Claude Fable 5 - High|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1074.9314240843462 }, { "id": "62a16b39127dec184d4b", "model": "GLM-5.1", "organization": "Z AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5634.406, "raw_score": 5634.406, "raw_scale": 1, "source_record": "GLM-5.1|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 333.6195100949586 }, { "id": "79e83343cacab72c9195", "model": "Gemini 3 Pro", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5478.158000000001, "raw_score": 5478.158000000001, "raw_scale": 1, "source_record": "Gemini 3 Pro|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 905.0284592342944 }, { "id": "6ff2a9d9b5ffbccf8df0", "model": "Claude Fable 5.1", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5421.556666666666, "raw_score": 5421.556666666666, "raw_scale": 1, "source_record": "Claude Fable 5.1|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1244.4364258428172 }, { "id": "cf38f80112b7be17ceff", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5396.418333333334, "raw_score": 5396.418333333334, "raw_scale": 1, "source_record": "Gemini 3.5 Flash|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1058.7341136231728 }, { "id": "87960e1af5e0393d7f53", "model": "Kimi K3 (Moonshot)", "organization": "Moonshot AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5165.035000000001, "raw_score": 5165.035000000001, "raw_scale": 1, "source_record": "Kimi K3 (Moonshot)|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 691.1590364826801 }, { "id": "a10bebdd7e740bffcf43", "model": "Qwen 3.6 Plus", "organization": "Alibaba", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5114.872000000001, "raw_score": 5114.872000000001, "raw_scale": 1, "source_record": "Qwen 3.6 Plus|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1097.1241807762697 }, { "id": "8bf6f70fc27574cde249", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5093.793333333332, "raw_score": 5093.793333333332, "raw_scale": 1, "source_record": "Gemini 3.8 Flash|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 490.1639138673693 }, { "id": "9db3e8f1d5cca8c8c1eb", "model": "Kimi K2.7 Code", "organization": "Moonshot AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5082.936, "raw_score": 5082.936, "raw_scale": 1, "source_record": "Kimi K2.7 Code|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1069.3686883465405 }, { "id": "687ef41a5669969e58d7", "model": "Claude Fable 5 - Low", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 5018.519999999999, "raw_score": 5018.519999999999, "raw_scale": 1, "source_record": "Claude Fable 5 - Low|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1225.2342459831914 }, { "id": "c164047af62f2caa71f4", "model": "Claude Opus 4.5", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 4967.056666666668, "raw_score": 4967.056666666668, "raw_scale": 1, "source_record": "Claude Opus 4.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 979.2482646760785 }, { "id": "0909d71380c5a23e2bca", "model": "Claude Fable 5 - Max", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 4966.639999999998, "raw_score": 4966.639999999998, "raw_scale": 1, "source_record": "Claude Fable 5 - Max|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 953.630613020576 }, { "id": "151ac538be4d4f31065a", "model": "Kimi K3 (Fireworks)", "organization": "Moonshot AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 4906.963333333332, "raw_score": 4906.963333333332, "raw_scale": 1, "source_record": "Kimi K3 (Fireworks)|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1652.4201746502342 }, { "id": "108af7eb2831b0ab5678", "model": "Grok 4.20", "organization": "xAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 4662.846666666667, "raw_score": 4662.846666666667, "raw_scale": 1, "source_record": "Grok 4.20|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1276.3313500515628 }, { "id": "25facecbbdb27ae1b47e", "model": "Claude Fable 5 - None", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 4529.944000000001, "raw_score": 4529.944000000001, "raw_scale": 1, "source_record": "Claude Fable 5 - None|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 662.3874858615611 }, { "id": "6140627815e42e8ad5ab", "model": "GLM-5", "organization": "Z AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 4432.122000000001, "raw_score": 4432.122000000001, "raw_scale": 1, "source_record": "GLM-5|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 393.5501827772413 }, { "id": "5e938c37fa6bca6d2ee4", "model": "Claude Fable 5 - Medium", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 4339.812500000001, "raw_score": 4339.812500000001, "raw_scale": 1, "source_record": "Claude Fable 5 - Medium|Vending-Bench 2; mean final money balance; USD", "n_samples": 4, "reported_stderr": 1426.8295361533499 }, { "id": "9c35db5b5e5a3295571d", "model": "Qwen 3.6 Max", "organization": "Alibaba", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 4254.1866666666665, "raw_score": 4254.1866666666665, "raw_scale": 1, "source_record": "Qwen 3.6 Max|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1469.5577387310937 }, { "id": "b600f342c98003909be5", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 4094.7120000000004, "raw_score": 4094.7120000000004, "raw_scale": 1, "source_record": "GPT-5.6 Luna|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1344.1559814113837 }, { "id": "bf45f2c9a34bde917ad7", "model": "Grok 4.5", "organization": "xAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 3887.4259999999986, "raw_score": 3887.4259999999986, "raw_scale": 1, "source_record": "Grok 4.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 435.2849462317763 }, { "id": "ccab7caed923d1f858e6", "model": "Claude Sonnet 4.5", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 3838.7400000000002, "raw_score": 3838.7400000000002, "raw_scale": 1, "source_record": "Claude Sonnet 4.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1102.0977149236812 }, { "id": "664a07d0e4d0304230e9", "model": "Gemini 3.1 Pro Custom Tools", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 3774.254, "raw_score": 3774.254, "raw_scale": 1, "source_record": "Gemini 3.1 Pro Custom Tools|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 1241.567713081328 }, { "id": "6a17bd62fb1eb03cc274", "model": "Gemini 3 Flash", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 3634.715999999999, "raw_score": 3634.715999999999, "raw_scale": 1, "source_record": "Gemini 3 Flash|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 639.3358759259487 }, { "id": "1125aca7a0001467fe40", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 3602.4633333333345, "raw_score": 3602.4633333333345, "raw_scale": 1, "source_record": "Claude Haiku 5.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 690.6447523952602 }, { "id": "994db2c135352ea67639", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 3591.3259999999996, "raw_score": 3591.3259999999996, "raw_scale": 1, "source_record": "GPT-5.2|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 392.0799849086917 }, { "id": "bf28f06ebe6f30bf92e7", "model": "Deepseek V4 Pro", "organization": "DeepSeek", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 3284.519999999999, "raw_score": 3284.519999999999, "raw_scale": 1, "source_record": "Deepseek V4 Pro|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 570.7043444960502 }, { "id": "945a11eb588e5af9b8ba", "model": "Claude Opus 4.8 - Max", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 2992.344, "raw_score": 2992.344, "raw_scale": 1, "source_record": "Claude Opus 4.8 - Max|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 722.0194956689185 }, { "id": "61fd347276cc6f53bcbe", "model": "GLM-4.7", "organization": "Z AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 2376.816, "raw_score": 2376.816, "raw_scale": 1, "source_record": "GLM-4.7|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 620.5982737858684 }, { "id": "a5f3202eed45ba743a6f", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 2157.7716666666665, "raw_score": 2157.7716666666665, "raw_scale": 1, "source_record": "MiniMax-M3|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 1151.0357691355398 }, { "id": "e58d4cb16b07a0a6b029", "model": "GPT-5.1", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 1473.432, "raw_score": 1473.432, "raw_scale": 1, "source_record": "GPT-5.1|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 756.1658962053234 }, { "id": "89d4c3514ee4adf25ecb", "model": "Kimi K2.5", "organization": "Moonshot AI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 1198.4579999999999, "raw_score": 1198.4579999999999, "raw_scale": 1, "source_record": "Kimi K2.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 596.6970510015948 }, { "id": "36810cab6ea5864db1f4", "model": "Grok 4.1 Fast", "organization": "xAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 1106.6280000000002, "raw_score": 1106.6280000000002, "raw_scale": 1, "source_record": "Grok 4.1 Fast|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 457.64703638721403 }, { "id": "e23e595dbc6e0b78d1a4", "model": "DeepSeek-V3.2", "organization": "DeepSeek", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 1034.0040000000001, "raw_score": 1034.0040000000001, "raw_scale": 1, "source_record": "DeepSeek-V3.2|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 503.1488034031285 }, { "id": "16d4b2080b9047632c6c", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 911.2139999999997, "raw_score": 911.2139999999997, "raw_scale": 1, "source_record": "Gemini 3.1 Pro|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 652.5516313718016 }, { "id": "39024e211f775b3ec00c", "model": "Gemini 2.5 Pro", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 573.6420000000003, "raw_score": 573.6420000000003, "raw_scale": 1, "source_record": "Gemini 2.5 Pro|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 465.30327731921267 }, { "id": "646198e65c4bacea7a50", "model": "Gemini 2.5 Flash", "organization": "Google", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 548.836, "raw_score": 548.836, "raw_scale": 1, "source_record": "Gemini 2.5 Flash|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 365.68944068157055 }, { "id": "57914692f2aaaf3b875f", "model": "Qwen 3.5 35B A3B", "organization": "Alibaba", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 462.68500000000023, "raw_score": 462.68500000000023, "raw_scale": 1, "source_record": "Qwen 3.5 35B A3B|Vending-Bench 2; mean final money balance; USD", "n_samples": 4, "reported_stderr": 486.4718261883213 }, { "id": "37bdf1e0768e0a975f32", "model": "Claude Haiku 4.5", "organization": "Anthropic", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 458.88599999999985, "raw_score": 458.88599999999985, "raw_scale": 1, "source_record": "Claude Haiku 4.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 274.9433748719906 }, { "id": "e4a8f2406197a359aafa", "model": "Qwen 3.5 27B", "organization": "Alibaba", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 201.97599999999994, "raw_score": 201.97599999999994, "raw_scale": 1, "source_record": "Qwen 3.5 27B|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 134.5486148795297 }, { "id": "2d3d624217bfdcf10757", "model": "MiniMax-M2", "organization": "MiniMax", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 160.5959999999999, "raw_score": 160.5959999999999, "raw_scale": 1, "source_record": "MiniMax-M2|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 97.3986709662919 }, { "id": "b9a2808ba7daa8107189", "model": "Qwen3 Max", "organization": "Alibaba", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 71.56500000000018, "raw_score": 71.56500000000018, "raw_scale": 1, "source_record": "Qwen3 Max|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 94.52113099725399 }, { "id": "f3ff3f3893e67a8b7995", "model": "Grok 4.3", "organization": "xAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 35.26166666666665, "raw_score": 35.26166666666665, "raw_scale": 1, "source_record": "Grok 4.3|Vending-Bench 2; mean final money balance; USD", "n_samples": 6, "reported_stderr": 58.90209913727844 }, { "id": "efaf4d071d2b62c631be", "model": "Qwen 3.5 Plus", "organization": "Alibaba", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": 0.5440000000000417, "raw_score": 0.5440000000000417, "raw_scale": 1, "source_record": "Qwen 3.5 Plus|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 20.30710703177587 }, { "id": "933dd3f55f93d9b237e4", "model": "Qwen3 235B A22B Thinking", "organization": "Alibaba", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": -11.340000000000078, "raw_score": -11.340000000000078, "raw_scale": 1, "source_record": "Qwen3 235B A22B Thinking|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 14.589143566364699 }, { "id": "72ba0dd4d03f1053c5e3", "model": "GPT-OSS-120b", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": -21.532000000000053, "raw_score": -21.532000000000053, "raw_scale": 1, "source_record": "GPT-OSS-120b|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 2.8717092471209504 }, { "id": "6cd83998ddd8b4788399", "model": "MiniMax-M2.5", "organization": "MiniMax", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": -23.15999999999999, "raw_score": -23.15999999999999, "raw_scale": 1, "source_record": "MiniMax-M2.5|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 0.2811761014026704 }, { "id": "4c0f4b870ef0efd6f602", "model": "GPT-5 mini", "organization": "OpenAI", "configuration": "Vending-Bench 2; mean final money balance; USD", "score": -31.18399999999995, "raw_score": -31.18399999999995, "raw_scale": 1, "source_record": "GPT-5 mini|Vending-Bench 2; mean final money balance; USD", "n_samples": 5, "reported_stderr": 5.972541837442429 } ] }, { "id": "swe-rebench", "name": "SWE-rebench (2026-05–06)", "group": "编程", "unit": "%", "metric": "Resolved rate", "protocol": { "en": "Official default window 2026-05-15 to 2026-07-01: 111 problems, 65 repositories; resolved rate (pass@1), not pass@5. Publisher-flagged contamination-risk rows are excluded.", "zh": "官方默认窗口 2026-05-15 至 2026-07-01，111 道题、65 个仓库；解决率（pass@1），不使用 pass@5；排除发布方标记有污染风险的行。" }, "source": { "name": "Nebius", "benchmark_creator": "Nebius", "url": "https://swe-rebench.com/", "data_url": "https://swe-rebench.com/", "sha256": "fbb1b659562843da266aa20ec2295253d727ccd9b9469c78ab32df6fad1fffed", "retrieved_at": "2026-11-02T03:34:31.687424+00:00", "selector": "default semantic leaderboard Resolved Rate; excludes contamination-risk rows", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "9e8b9e6f496b12d98e04", "model": "Junie", "organization": "Junie", "configuration": "Resolved rate; official default board; pass@1", "score": 0.618, "raw_score": 61.8, "raw_scale": 100, "source_record": "Junie|Resolved rate; official default board; pass@1" }, { "id": "5d0968c1a8a157847b66", "model": "Claude Code", "organization": "Anthropic", "configuration": "Resolved rate; official default board; pass@1", "score": 0.604, "raw_score": 60.4, "raw_scale": 100, "source_record": "Claude Code|Resolved rate; official default board; pass@1" }, { "id": "20f2ac72f605251d3f15", "model": "Codex", "organization": "OpenAI", "configuration": "Resolved rate; official default board; pass@1", "score": 0.58, "raw_score": 58.0, "raw_scale": 100, "source_record": "Codex|Resolved rate; official default board; pass@1" }, { "id": "e911de5af3ffea14e3ad", "model": "Cursor", "organization": "Cursor", "configuration": "Resolved rate; official default board; pass@1", "score": 0.517, "raw_score": 51.7, "raw_scale": 100, "source_record": "Cursor|Resolved rate; official default board; pass@1" }, { "id": "66b0e92af7e413d63beb", "model": "MiMo V2.5 Pro", "organization": "XiaomiMiMo", "configuration": "Resolved rate; official default board; pass@1", "score": 0.465, "raw_score": 46.5, "raw_scale": 100, "source_record": "MiMo V2.5 Pro|Resolved rate; official default board; pass@1" }, { "id": "80d87db49ba730ac8d40", "model": "DeepSeek-V4 Pro [high]", "organization": "DeepSeek", "configuration": "Resolved rate; official default board; pass@1", "score": 0.402, "raw_score": 40.2, "raw_scale": 100, "source_record": "DeepSeek-V4 Pro [high]|Resolved rate; official default board; pass@1" }, { "id": "426d21feb7dc87f969f1", "model": "Qwen3.6-27B", "organization": "Qwen", "configuration": "Resolved rate; official default board; pass@1", "score": 0.312, "raw_score": 31.2, "raw_scale": 100, "source_record": "Qwen3.6-27B|Resolved rate; official default board; pass@1" }, { "id": "81ec13cd5fd7a86be635", "model": "Qwen3.6-35B-A3B", "organization": "Qwen", "configuration": "Resolved rate; official default board; pass@1", "score": 0.247, "raw_score": 24.7, "raw_scale": 100, "source_record": "Qwen3.6-35B-A3B|Resolved rate; official default board; pass@1" }, { "id": "c466b04c5b72442f1b00", "model": "Qwen3.5-35B-A3B", "organization": "Qwen", "configuration": "Resolved rate; official default board; pass@1", "score": 0.171, "raw_score": 17.1, "raw_scale": 100, "source_record": "Qwen3.5-35B-A3B|Resolved rate; official default board; pass@1" } ] }, { "id": "multi-swe-bench-java", "name": "Multi-SWE-bench (Java)", "group": "编程", "unit": "%", "metric": "Resolved rate", "protocol": { "en": "Java Verified subset; verified leaderboard submissions, latest dated run per model/agent. Different language task sets are not averaged together.", "zh": "Java Verified 子集；采用榜单已验证提交，各模型与 Agent 使用最新日期记录。不同语言任务集不混合平均。" }, "source": { "name": "ByteDance Seed", "benchmark_creator": "ByteDance Seed", "url": "https://multi-swe-bench.github.io/", "data_url": "https://raw.githubusercontent.com/multi-swe-bench/experiments/refs/heads/dist/leaderboard.json", "sha256": "7ac5c3356b384b6eabe211b331ca72e7766832eca7a92ae954de09f893cacd72", "retrieved_at": "2026-11-02T03:36:07.507718+00:00", "selector": "Java/Verified: verified=true, resolvedRate; latest date per model+agent", "kind": "official_submissions", "status": "current_snapshot" }, "results": [ { "id": "cdd6976c75cf0da17177", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MSWE-agent", "score": 0.2890625, "raw_score": 0.2890625, "raw_scale": 1, "source_record": "java/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250426_MSWE-agent_Gemini-2.5-Pro" }, { "id": "05158d0af60c154123b2", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.234375, "raw_score": 0.234375, "raw_scale": 1, "source_record": "java/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MSWE-agent_Claude-3.7-Sonnet" }, { "id": "0d9cdac06e5c4d3ef474", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.2265625, "raw_score": 0.2265625, "raw_scale": 1, "source_record": "java/verified/20250329_MagentLess_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MagentLess_DeepSeek-R1" }, { "id": "ed7b0184ca6269420bd1", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.21875, "raw_score": 0.21875, "raw_scale": 1, "source_record": "java/verified/20250329_MopenHands_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MopenHands_Claude-3.7-Sonnet" }, { "id": "7e78cf7ef0d8e7bbd11e", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MagentLess", "score": 0.21875, "raw_score": 0.21875, "raw_scale": 1, "source_record": "java/verified/20250426_MagentLess_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250426_MagentLess_Gemini-2.5-Pro" }, { "id": "467bf9dddd55c001a0d1", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.21875, "raw_score": 0.21875, "raw_scale": 1, "source_record": "java/verified/20250329_MSWE-agent_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MSWE-agent_OpenAI-o1" }, { "id": "e59f527f8691bbc11e3a", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.2109375, "raw_score": 0.2109375, "raw_scale": 1, "source_record": "java/verified/20250329_MagentLess_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MagentLess_OpenAI-o1" }, { "id": "d83b7b5e2a9548fdc74a", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.203125, "raw_score": 0.203125, "raw_scale": 1, "source_record": "java/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)" }, { "id": "8a1b0a01104aff29e579", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.1640625, "raw_score": 0.1640625, "raw_scale": 1, "source_record": "java/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MSWE-agent_OpenAI-o3-mini-high" }, { "id": "a593e3b27b0512d119dd", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.1484375, "raw_score": 0.1484375, "raw_scale": 1, "source_record": "java/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)" }, { "id": "597a12b00158e4a35494", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.1484375, "raw_score": 0.1484375, "raw_scale": 1, "source_record": "java/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)" }, { "id": "98a200e769fc5ed5e5c4", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MagentLess", "score": 0.1484375, "raw_score": 0.1484375, "raw_scale": 1, "source_record": "java/verified/20250426_MagentLess_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250426_MagentLess_Llama-4-Maverick" }, { "id": "5fe0c822f99bc6c0f163", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.140625, "raw_score": 0.140625, "raw_scale": 1, "source_record": "java/verified/20250329_MagentLess_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MagentLess_Claude-3.7-Sonnet" }, { "id": "83b45bc1d4774159d118", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MagentLess", "score": 0.1328125, "raw_score": 0.1328125, "raw_scale": 1, "source_record": "java/verified/20250426_MagentLess_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250426_MagentLess_Doubao-1.5-thinking" }, { "id": "e17723f36744ede3b06d", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.125, "raw_score": 0.125, "raw_scale": 1, "source_record": "java/verified/20250329_MSWE-agent_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MSWE-agent_GPT-4o-1120" }, { "id": "7c05738178c5634cf4c9", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MopenHands", "score": 0.125, "raw_score": 0.125, "raw_scale": 1, "source_record": "java/verified/20250426_MopenHands_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250426_MopenHands_Gemini-2.5-Pro" }, { "id": "0092fea3f23227924c9e", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.1171875, "raw_score": 0.1171875, "raw_scale": 1, "source_record": "java/verified/20250329_MSWE-agent_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MSWE-agent_DeepSeek-V3" }, { "id": "7a2d007224c5ba3b278a", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.1171875, "raw_score": 0.1171875, "raw_scale": 1, "source_record": "java/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250426_MSWE-agent_Doubao-1.5-thinking" }, { "id": "8f2d3fbae486dab89b6b", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.1171875, "raw_score": 0.1171875, "raw_scale": 1, "source_record": "java/verified/20250329_MagentLess_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MagentLess_GPT-4o-1120" }, { "id": "a109a16d1ce893c76b8c", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MopenHands", "score": 0.109375, "raw_score": 0.109375, "raw_scale": 1, "source_record": "java/verified/20250426_MopenHands_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250426_MopenHands_Doubao-1.5-thinking" }, { "id": "9beaa797cf296bb004e5", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MagentLess", "score": 0.109375, "raw_score": 0.109375, "raw_scale": 1, "source_record": "java/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MagentLess_Qwen2.5-72B-Instruct" }, { "id": "feff435207e0c91254b7", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.1015625, "raw_score": 0.1015625, "raw_scale": 1, "source_record": "java/verified/20250329_MopenHands_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MopenHands_OpenAI-o3-mini-high" }, { "id": "644cea614ee9aaa57229", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.09375, "raw_score": 0.09375, "raw_scale": 1, "source_record": "java/verified/20250329_MSWE-agent_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MSWE-agent_DeepSeek-R1" }, { "id": "687052d671ee601f1df2", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.09375, "raw_score": 0.09375, "raw_scale": 1, "source_record": "java/verified/20250329_MopenHands_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MopenHands_DeepSeek-V3" }, { "id": "2348635fb12fd35b7c94", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.09375, "raw_score": 0.09375, "raw_scale": 1, "source_record": "java/verified/20250329_MopenHands_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MopenHands_GPT-4o-1120" }, { "id": "bfc6259edfc38b10606c", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.0859375, "raw_score": 0.0859375, "raw_scale": 1, "source_record": "java/verified/20250329_MopenHands_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MopenHands_DeepSeek-R1" }, { "id": "ded192545f9c10e238d4", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.0703125, "raw_score": 0.0703125, "raw_scale": 1, "source_record": "java/verified/20250329_MagentLess_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MagentLess_DeepSeek-V3" }, { "id": "13890daf130738773a64", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.0703125, "raw_score": 0.0703125, "raw_scale": 1, "source_record": "java/verified/20250329_MSWE-agent_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MSWE-agent_Doubao-1.5-pro" }, { "id": "cd58f4d180101186e952", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MSWE-agent", "score": 0.0625, "raw_score": 0.0625, "raw_scale": 1, "source_record": "java/verified/20250426_MSWE-agent_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250426_MSWE-agent_Llama-4-Maverick" }, { "id": "486d55ccf88a5cfdff0b", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MopenHands", "score": 0.0625, "raw_score": 0.0625, "raw_scale": 1, "source_record": "java/verified/20250426_MopenHands_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250426_MopenHands_Llama-4-Maverick" }, { "id": "88d6fb543599750d1b99", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MagentLess", "score": 0.0546875, "raw_score": 0.0546875, "raw_scale": 1, "source_record": "java/verified/20250329_MagentLess_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MagentLess_Doubao-1.5-pro" }, { "id": "95f7ac86b2f1cf3fc222", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.0546875, "raw_score": 0.0546875, "raw_scale": 1, "source_record": "java/verified/20250329_MagentLess_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MagentLess_OpenAI-o3-mini-high" }, { "id": "dd8d55c60db44c95ec8e", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.0390625, "raw_score": 0.0390625, "raw_scale": 1, "source_record": "java/verified/20250329_MopenHands_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MopenHands_OpenAI-o1" }, { "id": "1399e0788086cd5a3df8", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MopenHands", "score": 0.03125, "raw_score": 0.03125, "raw_scale": 1, "source_record": "java/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MopenHands_Qwen2.5-72B-Instruct" }, { "id": "add50cf32be717444298", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MSWE-agent", "score": 0.0234375, "raw_score": 0.0234375, "raw_scale": 1, "source_record": "java/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct" }, { "id": "d955b2fd8eaf10b6222d", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MopenHands", "score": 0.0078125, "raw_score": 0.0078125, "raw_scale": 1, "source_record": "java/verified/20250329_MopenHands_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/java/verified/20250329_MopenHands_Doubao-1.5-pro" } ] }, { "id": "multi-swe-bench-typescript", "name": "Multi-SWE-bench (TypeScript)", "group": "编程", "unit": "%", "metric": "Resolved rate", "protocol": { "en": "TypeScript Verified subset; verified leaderboard submissions, latest dated run per model/agent. Different language task sets are not averaged together.", "zh": "TypeScript Verified 子集；采用榜单已验证提交，各模型与 Agent 使用最新日期记录。不同语言任务集不混合平均。" }, "source": { "name": "ByteDance Seed", "benchmark_creator": "ByteDance Seed", "url": "https://multi-swe-bench.github.io/", "data_url": "https://raw.githubusercontent.com/multi-swe-bench/experiments/refs/heads/dist/leaderboard.json", "sha256": "7ac5c3356b384b6eabe211b331ca72e7766832eca7a92ae954de09f893cacd72", "retrieved_at": "2026-11-02T03:36:07.507718+00:00", "selector": "TypeScript/Verified: verified=true, resolvedRate; latest date per model+agent", "kind": "official_submissions", "status": "current_snapshot" }, "results": [ { "id": "204a5bc9a14048ec6ccf", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: RepoRepair", "score": 0.27232142857142855, "raw_score": 0.27232142857142855, "raw_scale": 1, "source_record": "typescript/verified/20250910_RepoRepair_Claude-3.5-Sonnet(Oct)", "source_date": "2025-09-10", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250910_RepoRepair_Claude-3.5-Sonnet(Oct)" }, { "id": "09704f349695cf70beed", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MopenHands", "score": 0.22321428571428573, "raw_score": 0.22321428571428573, "raw_scale": 1, "source_record": "typescript/verified/20250426_MopenHands_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250426_MopenHands_Gemini-2.5-Pro" }, { "id": "ec10a5ded30849362a15", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.11607142857142858, "raw_score": 0.11607142857142858, "raw_scale": 1, "source_record": "typescript/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)" }, { "id": "c3a6da4a018cd07ffe2e", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MagentLess", "score": 0.11607142857142858, "raw_score": 0.11607142857142858, "raw_scale": 1, "source_record": "typescript/verified/20250426_MagentLess_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250426_MagentLess_Gemini-2.5-Pro" }, { "id": "5697a67a6e3a765d741c", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.11160714285714286, "raw_score": 0.11160714285714286, "raw_scale": 1, "source_record": "typescript/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MSWE-agent_Claude-3.7-Sonnet" }, { "id": "550b758d1a559d3e5935", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MagentLess", "score": 0.09375, "raw_score": 0.09375, "raw_scale": 1, "source_record": "typescript/verified/20250426_MagentLess_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250426_MagentLess_Llama-4-Maverick" }, { "id": "5f12d71268f2098b398e", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MSWE-agent", "score": 0.08928571428571429, "raw_score": 0.08928571428571429, "raw_scale": 1, "source_record": "typescript/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250426_MSWE-agent_Gemini-2.5-Pro" }, { "id": "848d800b1d30b0d40776", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.08035714285714286, "raw_score": 0.08035714285714286, "raw_scale": 1, "source_record": "typescript/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)" }, { "id": "7f6643749ddb613eabab", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MopenHands", "score": 0.08035714285714286, "raw_score": 0.08035714285714286, "raw_scale": 1, "source_record": "typescript/verified/20250426_MopenHands_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250426_MopenHands_Llama-4-Maverick" }, { "id": "0caa638dcca203f365f2", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MagentLess", "score": 0.07589285714285714, "raw_score": 0.07589285714285714, "raw_scale": 1, "source_record": "typescript/verified/20250426_MagentLess_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250426_MagentLess_Doubao-1.5-thinking" }, { "id": "b5fb41b9ffcb738030de", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.07142857142857142, "raw_score": 0.07142857142857142, "raw_scale": 1, "source_record": "typescript/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250426_MSWE-agent_Doubao-1.5-thinking" }, { "id": "1a2d879b1634a1f55312", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.06696428571428571, "raw_score": 0.06696428571428571, "raw_scale": 1, "source_record": "typescript/verified/20250329_MagentLess_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MagentLess_DeepSeek-V3" }, { "id": "5903b5df46d5e516fc0d", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.0625, "raw_score": 0.0625, "raw_scale": 1, "source_record": "typescript/verified/20250329_MagentLess_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MagentLess_DeepSeek-R1" }, { "id": "eb37728956ed1d98f572", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.05803571428571429, "raw_score": 0.05803571428571429, "raw_scale": 1, "source_record": "typescript/verified/20250329_MSWE-agent_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MSWE-agent_DeepSeek-R1" }, { "id": "5fe5137498d6e4829463", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.05803571428571429, "raw_score": 0.05803571428571429, "raw_scale": 1, "source_record": "typescript/verified/20250329_MagentLess_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MagentLess_OpenAI-o1" }, { "id": "468740c5cbb54c5ada1b", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MopenHands", "score": 0.05357142857142857, "raw_score": 0.05357142857142857, "raw_scale": 1, "source_record": "typescript/verified/20250426_MopenHands_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250426_MopenHands_Doubao-1.5-thinking" }, { "id": "f228882f33cacb0b571e", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.049107142857142856, "raw_score": 0.049107142857142856, "raw_scale": 1, "source_record": "typescript/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)" }, { "id": "2fb75e9dfa4c279f1355", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.049107142857142856, "raw_score": 0.049107142857142856, "raw_scale": 1, "source_record": "typescript/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MSWE-agent_OpenAI-o3-mini-high" }, { "id": "2a34d6da9153fe9431de", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MSWE-agent", "score": 0.044642857142857144, "raw_score": 0.044642857142857144, "raw_scale": 1, "source_record": "typescript/verified/20250426_MSWE-agent_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250426_MSWE-agent_Llama-4-Maverick" }, { "id": "395a83a31f748249d6a3", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MagentLess", "score": 0.044642857142857144, "raw_score": 0.044642857142857144, "raw_scale": 1, "source_record": "typescript/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MagentLess_Qwen2.5-72B-Instruct" }, { "id": "83b2924ea4126981bdd2", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.04017857142857143, "raw_score": 0.04017857142857143, "raw_scale": 1, "source_record": "typescript/verified/20250329_MSWE-agent_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MSWE-agent_OpenAI-o1" }, { "id": "c73d6be08e87c9e1e6bc", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.03571428571428571, "raw_score": 0.03571428571428571, "raw_scale": 1, "source_record": "typescript/verified/20250329_MagentLess_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MagentLess_Claude-3.7-Sonnet" }, { "id": "5f7bf1941617cbc82943", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.026785714285714284, "raw_score": 0.026785714285714284, "raw_scale": 1, "source_record": "typescript/verified/20250329_MSWE-agent_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MSWE-agent_DeepSeek-V3" }, { "id": "cde7f11a2cc4e3bf0ae0", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.022321428571428572, "raw_score": 0.022321428571428572, "raw_scale": 1, "source_record": "typescript/verified/20250329_MopenHands_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MopenHands_Claude-3.7-Sonnet" }, { "id": "ee1fdd76caabc6020ee4", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MagentLess", "score": 0.022321428571428572, "raw_score": 0.022321428571428572, "raw_scale": 1, "source_record": "typescript/verified/20250329_MagentLess_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MagentLess_Doubao-1.5-pro" }, { "id": "f090bfd7be5b191633c5", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.022321428571428572, "raw_score": 0.022321428571428572, "raw_scale": 1, "source_record": "typescript/verified/20250329_MagentLess_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MagentLess_GPT-4o-1120" }, { "id": "58535ba8a12cae2228ff", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.017857142857142856, "raw_score": 0.017857142857142856, "raw_scale": 1, "source_record": "typescript/verified/20250329_MSWE-agent_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MSWE-agent_Doubao-1.5-pro" }, { "id": "d0f607e06cf7b4cafbee", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.013392857142857142, "raw_score": 0.013392857142857142, "raw_scale": 1, "source_record": "typescript/verified/20250329_MopenHands_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MopenHands_DeepSeek-V3" }, { "id": "b0d74a17ef18efe5f381", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.004464285714285714, "raw_score": 0.004464285714285714, "raw_scale": 1, "source_record": "typescript/verified/20250329_MopenHands_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MopenHands_DeepSeek-R1" }, { "id": "caeafda2978a9ad47f60", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.004464285714285714, "raw_score": 0.004464285714285714, "raw_scale": 1, "source_record": "typescript/verified/20250329_MSWE-agent_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MSWE-agent_GPT-4o-1120" }, { "id": "05790545b1eae77da88d", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.004464285714285714, "raw_score": 0.004464285714285714, "raw_scale": 1, "source_record": "typescript/verified/20250329_MopenHands_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MopenHands_OpenAI-o1" }, { "id": "1aab4b7c68dc771fdf98", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.004464285714285714, "raw_score": 0.004464285714285714, "raw_scale": 1, "source_record": "typescript/verified/20250329_MagentLess_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MagentLess_OpenAI-o3-mini-high" }, { "id": "dfa6c5cd3f3d38b5b9e9", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.004464285714285714, "raw_score": 0.004464285714285714, "raw_scale": 1, "source_record": "typescript/verified/20250329_MopenHands_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MopenHands_OpenAI-o3-mini-high" }, { "id": "9c51b970bd3363728fb4", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MopenHands", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "typescript/verified/20250329_MopenHands_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MopenHands_Doubao-1.5-pro" }, { "id": "5981befd4641e99bc570", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "typescript/verified/20250329_MopenHands_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MopenHands_GPT-4o-1120" }, { "id": "fd235b103e8e2eba58e2", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MSWE-agent", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "typescript/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct" }, { "id": "6458d85288b5e4ce700e", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MopenHands", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "typescript/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/typescript/verified/20250329_MopenHands_Qwen2.5-72B-Instruct" } ] }, { "id": "multi-swe-bench-javascript", "name": "Multi-SWE-bench (JavaScript)", "group": "编程", "unit": "%", "metric": "Resolved rate", "protocol": { "en": "JavaScript Verified subset; verified leaderboard submissions, latest dated run per model/agent. Different language task sets are not averaged together.", "zh": "JavaScript Verified 子集；采用榜单已验证提交，各模型与 Agent 使用最新日期记录。不同语言任务集不混合平均。" }, "source": { "name": "ByteDance Seed", "benchmark_creator": "ByteDance Seed", "url": "https://multi-swe-bench.github.io/", "data_url": "https://raw.githubusercontent.com/multi-swe-bench/experiments/refs/heads/dist/leaderboard.json", "sha256": "7ac5c3356b384b6eabe211b331ca72e7766832eca7a92ae954de09f893cacd72", "retrieved_at": "2026-11-02T03:36:07.507718+00:00", "selector": "JavaScript/Verified: verified=true, resolvedRate; latest date per model+agent", "kind": "official_submissions", "status": "current_snapshot" }, "results": [ { "id": "3910e8a684fb63d1c81a", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MopenHands", "score": 0.16292134831460675, "raw_score": 0.16292134831460675, "raw_scale": 1, "source_record": "javascript/verified/20250426_MopenHands_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250426_MopenHands_Gemini-2.5-Pro" }, { "id": "102a5062a03f5553578b", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MopenHands", "score": 0.09550561797752809, "raw_score": 0.09550561797752809, "raw_scale": 1, "source_record": "javascript/verified/20250426_MopenHands_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250426_MopenHands_Doubao-1.5-thinking" }, { "id": "1f96e28acc9545d3cf8a", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MagentLess", "score": 0.08707865168539326, "raw_score": 0.08707865168539326, "raw_scale": 1, "source_record": "javascript/verified/20250426_MagentLess_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250426_MagentLess_Gemini-2.5-Pro" }, { "id": "b91482a7229210f64596", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MSWE-agent", "score": 0.07584269662921349, "raw_score": 0.07584269662921349, "raw_scale": 1, "source_record": "javascript/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250426_MSWE-agent_Gemini-2.5-Pro" }, { "id": "fb41c05159959c2c1654", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MagentLess", "score": 0.056179775280898875, "raw_score": 0.056179775280898875, "raw_scale": 1, "source_record": "javascript/verified/20250426_MagentLess_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250426_MagentLess_Doubao-1.5-thinking" }, { "id": "17dc2d36a022cb8a9342", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.05056179775280899, "raw_score": 0.05056179775280899, "raw_scale": 1, "source_record": "javascript/verified/20250329_MopenHands_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MopenHands_Claude-3.7-Sonnet" }, { "id": "36cb01b20b05a31c58b1", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MagentLess", "score": 0.05056179775280899, "raw_score": 0.05056179775280899, "raw_scale": 1, "source_record": "javascript/verified/20250426_MagentLess_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250426_MagentLess_Llama-4-Maverick" }, { "id": "6142e08b64a115b972fd", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.05056179775280899, "raw_score": 0.05056179775280899, "raw_scale": 1, "source_record": "javascript/verified/20250329_MagentLess_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MagentLess_OpenAI-o1" }, { "id": "bcc8c3682e2e24ffc178", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.047752808988764044, "raw_score": 0.047752808988764044, "raw_scale": 1, "source_record": "javascript/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MSWE-agent_Claude-3.7-Sonnet" }, { "id": "d5f7b2a94fa0f8958c28", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MopenHands", "score": 0.047752808988764044, "raw_score": 0.047752808988764044, "raw_scale": 1, "source_record": "javascript/verified/20250426_MopenHands_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250426_MopenHands_Llama-4-Maverick" }, { "id": "12a7712e20e1655402dc", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.0449438202247191, "raw_score": 0.0449438202247191, "raw_scale": 1, "source_record": "javascript/verified/20250329_MagentLess_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MagentLess_DeepSeek-R1" }, { "id": "fba6876215b20e270408", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.042134831460674156, "raw_score": 0.042134831460674156, "raw_scale": 1, "source_record": "javascript/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)" }, { "id": "342eeb989bb8b60d8a36", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.042134831460674156, "raw_score": 0.042134831460674156, "raw_scale": 1, "source_record": "javascript/verified/20250329_MSWE-agent_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MSWE-agent_OpenAI-o1" }, { "id": "c580c7c88f7009e24880", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.042134831460674156, "raw_score": 0.042134831460674156, "raw_scale": 1, "source_record": "javascript/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MSWE-agent_OpenAI-o3-mini-high" }, { "id": "01812b1ae344a84fa95e", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.03651685393258427, "raw_score": 0.03651685393258427, "raw_scale": 1, "source_record": "javascript/verified/20250329_MopenHands_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MopenHands_OpenAI-o1" }, { "id": "0269fbd438cfdd271c0a", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.033707865168539325, "raw_score": 0.033707865168539325, "raw_scale": 1, "source_record": "javascript/verified/20250329_MagentLess_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MagentLess_DeepSeek-V3" }, { "id": "b5f9a51525d4e95f7905", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.033707865168539325, "raw_score": 0.033707865168539325, "raw_scale": 1, "source_record": "javascript/verified/20250329_MopenHands_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MopenHands_OpenAI-o3-mini-high" }, { "id": "5b7ae8a06a97c7a13eeb", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.028089887640449437, "raw_score": 0.028089887640449437, "raw_scale": 1, "source_record": "javascript/verified/20250329_MagentLess_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MagentLess_OpenAI-o3-mini-high" }, { "id": "1724fc281f9a569a9c4f", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.025280898876404494, "raw_score": 0.025280898876404494, "raw_scale": 1, "source_record": "javascript/verified/20250329_MopenHands_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MopenHands_DeepSeek-R1" }, { "id": "21c2335171a6c515260e", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.025280898876404494, "raw_score": 0.025280898876404494, "raw_scale": 1, "source_record": "javascript/verified/20250329_MSWE-agent_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MSWE-agent_DeepSeek-V3" }, { "id": "f10a68477d2331ed6c31", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MSWE-agent", "score": 0.02247191011235955, "raw_score": 0.02247191011235955, "raw_scale": 1, "source_record": "javascript/verified/20250426_MSWE-agent_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250426_MSWE-agent_Llama-4-Maverick" }, { "id": "acb8360ed5e342c0f326", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.019662921348314606, "raw_score": 0.019662921348314606, "raw_scale": 1, "source_record": "javascript/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)" }, { "id": "61e3c471c91bb1206c9e", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.019662921348314606, "raw_score": 0.019662921348314606, "raw_scale": 1, "source_record": "javascript/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)" }, { "id": "af4f44f8e22393c8dfdf", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.019662921348314606, "raw_score": 0.019662921348314606, "raw_scale": 1, "source_record": "javascript/verified/20250329_MagentLess_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MagentLess_Claude-3.7-Sonnet" }, { "id": "cd4692b0388cdf5853fd", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.019662921348314606, "raw_score": 0.019662921348314606, "raw_scale": 1, "source_record": "javascript/verified/20250329_MopenHands_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MopenHands_GPT-4o-1120" }, { "id": "3bb9dbe49031bea7cee4", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.016853932584269662, "raw_score": 0.016853932584269662, "raw_scale": 1, "source_record": "javascript/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250426_MSWE-agent_Doubao-1.5-thinking" }, { "id": "6f335445fb58ec3bc504", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.014044943820224719, "raw_score": 0.014044943820224719, "raw_scale": 1, "source_record": "javascript/verified/20250329_MSWE-agent_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MSWE-agent_DeepSeek-R1" }, { "id": "1154c71f69855abd8aba", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.014044943820224719, "raw_score": 0.014044943820224719, "raw_scale": 1, "source_record": "javascript/verified/20250329_MSWE-agent_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MSWE-agent_Doubao-1.5-pro" }, { "id": "93e0a18cb0c5911be61e", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.014044943820224719, "raw_score": 0.014044943820224719, "raw_scale": 1, "source_record": "javascript/verified/20250329_MagentLess_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MagentLess_GPT-4o-1120" }, { "id": "4a325c2fdfe5538a4a70", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.011235955056179775, "raw_score": 0.011235955056179775, "raw_scale": 1, "source_record": "javascript/verified/20250329_MopenHands_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MopenHands_DeepSeek-V3" }, { "id": "d2383eac6cc13dee15bb", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MagentLess", "score": 0.011235955056179775, "raw_score": 0.011235955056179775, "raw_scale": 1, "source_record": "javascript/verified/20250329_MagentLess_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MagentLess_Doubao-1.5-pro" }, { "id": "687141bf26915c2f4dea", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MopenHands", "score": 0.011235955056179775, "raw_score": 0.011235955056179775, "raw_scale": 1, "source_record": "javascript/verified/20250329_MopenHands_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MopenHands_Doubao-1.5-pro" }, { "id": "b6bebacbe4afcd275884", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.008426966292134831, "raw_score": 0.008426966292134831, "raw_scale": 1, "source_record": "javascript/verified/20250329_MSWE-agent_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MSWE-agent_GPT-4o-1120" }, { "id": "39838deb1de5aea2e8c2", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MagentLess", "score": 0.008426966292134831, "raw_score": 0.008426966292134831, "raw_scale": 1, "source_record": "javascript/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MagentLess_Qwen2.5-72B-Instruct" }, { "id": "f753690eb2c921da21d7", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MopenHands", "score": 0.008426966292134831, "raw_score": 0.008426966292134831, "raw_scale": 1, "source_record": "javascript/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MopenHands_Qwen2.5-72B-Instruct" }, { "id": "57c2f21fb5f9a288b17e", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MSWE-agent", "score": 0.0056179775280898875, "raw_score": 0.0056179775280898875, "raw_scale": 1, "source_record": "javascript/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/javascript/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct" } ] }, { "id": "multi-swe-bench-go", "name": "Multi-SWE-bench (Go)", "group": "编程", "unit": "%", "metric": "Resolved rate", "protocol": { "en": "Go Verified subset; verified leaderboard submissions, latest dated run per model/agent. Different language task sets are not averaged together.", "zh": "Go Verified 子集；采用榜单已验证提交，各模型与 Agent 使用最新日期记录。不同语言任务集不混合平均。" }, "source": { "name": "ByteDance Seed", "benchmark_creator": "ByteDance Seed", "url": "https://multi-swe-bench.github.io/", "data_url": "https://raw.githubusercontent.com/multi-swe-bench/experiments/refs/heads/dist/leaderboard.json", "sha256": "7ac5c3356b384b6eabe211b331ca72e7766832eca7a92ae954de09f893cacd72", "retrieved_at": "2026-11-02T03:36:07.507718+00:00", "selector": "Go/Verified: verified=true, resolvedRate; latest date per model+agent", "kind": "official_submissions", "status": "current_snapshot" }, "results": [ { "id": "e7c5363abd41193c2f84", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MopenHands", "score": 0.1261682242990654, "raw_score": 0.1261682242990654, "raw_scale": 1, "source_record": "go/verified/20250426_MopenHands_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250426_MopenHands_Gemini-2.5-Pro" }, { "id": "9164e53aa6e28775c559", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MSWE-agent", "score": 0.09813084112149532, "raw_score": 0.09813084112149532, "raw_scale": 1, "source_record": "go/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250426_MSWE-agent_Gemini-2.5-Pro" }, { "id": "f25257dab8747f907c89", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.07476635514018691, "raw_score": 0.07476635514018691, "raw_scale": 1, "source_record": "go/verified/20250329_MopenHands_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MopenHands_Claude-3.7-Sonnet" }, { "id": "009932e731c504445b78", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.06775700934579439, "raw_score": 0.06775700934579439, "raw_scale": 1, "source_record": "go/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)" }, { "id": "eb2a98fa3a85e21c2508", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MopenHands", "score": 0.06074766355140187, "raw_score": 0.06074766355140187, "raw_scale": 1, "source_record": "go/verified/20250426_MopenHands_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250426_MopenHands_Doubao-1.5-thinking" }, { "id": "3f89e4819df41fd06ee7", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MagentLess", "score": 0.06074766355140187, "raw_score": 0.06074766355140187, "raw_scale": 1, "source_record": "go/verified/20250426_MagentLess_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250426_MagentLess_Gemini-2.5-Pro" }, { "id": "3b431f7c9577e5ad6103", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.05841121495327103, "raw_score": 0.05841121495327103, "raw_scale": 1, "source_record": "go/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)" }, { "id": "42eea0826b81831775ba", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.05841121495327103, "raw_score": 0.05841121495327103, "raw_scale": 1, "source_record": "go/verified/20250329_MagentLess_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MagentLess_Claude-3.7-Sonnet" }, { "id": "0696d9a7ebd9f67728b9", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MopenHands", "score": 0.056074766355140186, "raw_score": 0.056074766355140186, "raw_scale": 1, "source_record": "go/verified/20250426_MopenHands_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250426_MopenHands_Llama-4-Maverick" }, { "id": "980f007eea26db91c58e", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.053738317757009345, "raw_score": 0.053738317757009345, "raw_scale": 1, "source_record": "go/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MSWE-agent_Claude-3.7-Sonnet" }, { "id": "ea2c627f19a366b96507", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.053738317757009345, "raw_score": 0.053738317757009345, "raw_scale": 1, "source_record": "go/verified/20250329_MagentLess_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MagentLess_DeepSeek-V3" }, { "id": "e93401ba189acb38e684", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.0514018691588785, "raw_score": 0.0514018691588785, "raw_scale": 1, "source_record": "go/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)" }, { "id": "da96e9c05e90d88a3dba", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.04672897196261682, "raw_score": 0.04672897196261682, "raw_scale": 1, "source_record": "go/verified/20250329_MSWE-agent_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MSWE-agent_OpenAI-o1" }, { "id": "de891f6915cd9c758163", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.04439252336448598, "raw_score": 0.04439252336448598, "raw_scale": 1, "source_record": "go/verified/20250329_MSWE-agent_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MSWE-agent_DeepSeek-V3" }, { "id": "7dc753605a62ed4e5cbe", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MagentLess", "score": 0.04439252336448598, "raw_score": 0.04439252336448598, "raw_scale": 1, "source_record": "go/verified/20250426_MagentLess_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250426_MagentLess_Doubao-1.5-thinking" }, { "id": "8b48b42057d29d76ef9a", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.04439252336448598, "raw_score": 0.04439252336448598, "raw_scale": 1, "source_record": "go/verified/20250329_MagentLess_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MagentLess_OpenAI-o1" }, { "id": "5ec409d79b9677f424a7", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.04205607476635514, "raw_score": 0.04205607476635514, "raw_scale": 1, "source_record": "go/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250426_MSWE-agent_Doubao-1.5-thinking" }, { "id": "dd2630b0843b3ca13cb6", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.0397196261682243, "raw_score": 0.0397196261682243, "raw_scale": 1, "source_record": "go/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MSWE-agent_OpenAI-o3-mini-high" }, { "id": "1beec267f44912cbda31", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.0397196261682243, "raw_score": 0.0397196261682243, "raw_scale": 1, "source_record": "go/verified/20250329_MagentLess_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MagentLess_OpenAI-o3-mini-high" }, { "id": "e907a23530612d68ac01", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.037383177570093455, "raw_score": 0.037383177570093455, "raw_scale": 1, "source_record": "go/verified/20250329_MagentLess_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MagentLess_DeepSeek-R1" }, { "id": "49feb20c860b9f61d4c2", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.037383177570093455, "raw_score": 0.037383177570093455, "raw_scale": 1, "source_record": "go/verified/20250329_MopenHands_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MopenHands_OpenAI-o1" }, { "id": "d37eb903e786e1251b20", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.035046728971962614, "raw_score": 0.035046728971962614, "raw_scale": 1, "source_record": "go/verified/20250329_MopenHands_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MopenHands_GPT-4o-1120" }, { "id": "c83da5df80612362a577", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MagentLess", "score": 0.035046728971962614, "raw_score": 0.035046728971962614, "raw_scale": 1, "source_record": "go/verified/20250426_MagentLess_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250426_MagentLess_Llama-4-Maverick" }, { "id": "d70dee2a8fb55b2fb6e2", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.028037383177570093, "raw_score": 0.028037383177570093, "raw_scale": 1, "source_record": "go/verified/20250329_MagentLess_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MagentLess_GPT-4o-1120" }, { "id": "1890ecb067d671316339", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MSWE-agent", "score": 0.028037383177570093, "raw_score": 0.028037383177570093, "raw_scale": 1, "source_record": "go/verified/20250426_MSWE-agent_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250426_MSWE-agent_Llama-4-Maverick" }, { "id": "23477f299f1a37996ad5", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.02336448598130841, "raw_score": 0.02336448598130841, "raw_scale": 1, "source_record": "go/verified/20250329_MSWE-agent_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MSWE-agent_GPT-4o-1120" }, { "id": "a9674165b0235f238bda", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.02336448598130841, "raw_score": 0.02336448598130841, "raw_scale": 1, "source_record": "go/verified/20250329_MopenHands_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MopenHands_OpenAI-o3-mini-high" }, { "id": "5e10813026be33fddc9b", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.02102803738317757, "raw_score": 0.02102803738317757, "raw_scale": 1, "source_record": "go/verified/20250329_MSWE-agent_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MSWE-agent_DeepSeek-R1" }, { "id": "df07b461c2b78a5f5859", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.02102803738317757, "raw_score": 0.02102803738317757, "raw_scale": 1, "source_record": "go/verified/20250329_MSWE-agent_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MSWE-agent_Doubao-1.5-pro" }, { "id": "91b6e1b0aa642b333eb3", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MagentLess", "score": 0.02102803738317757, "raw_score": 0.02102803738317757, "raw_scale": 1, "source_record": "go/verified/20250329_MagentLess_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MagentLess_Doubao-1.5-pro" }, { "id": "64cbd28f6252cfa43429", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MopenHands", "score": 0.016355140186915886, "raw_score": 0.016355140186915886, "raw_scale": 1, "source_record": "go/verified/20250329_MopenHands_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MopenHands_Doubao-1.5-pro" }, { "id": "b77cb9f93d06e9b0dd67", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MagentLess", "score": 0.014018691588785047, "raw_score": 0.014018691588785047, "raw_scale": 1, "source_record": "go/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MagentLess_Qwen2.5-72B-Instruct" }, { "id": "3d225964cd7d5f45369f", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MopenHands", "score": 0.014018691588785047, "raw_score": 0.014018691588785047, "raw_scale": 1, "source_record": "go/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MopenHands_Qwen2.5-72B-Instruct" }, { "id": "e8065c48369eeb39b13b", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.007009345794392523, "raw_score": 0.007009345794392523, "raw_scale": 1, "source_record": "go/verified/20250329_MopenHands_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MopenHands_DeepSeek-V3" }, { "id": "7f38177bfd601c25c623", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MSWE-agent", "score": 0.004672897196261682, "raw_score": 0.004672897196261682, "raw_scale": 1, "source_record": "go/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct" }, { "id": "ed98ad1403cb2a57bb20", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "go/verified/20250329_MopenHands_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/go/verified/20250329_MopenHands_DeepSeek-R1" } ] }, { "id": "multi-swe-bench-rust", "name": "Multi-SWE-bench (Rust)", "group": "编程", "unit": "%", "metric": "Resolved rate", "protocol": { "en": "Rust Verified subset; verified leaderboard submissions, latest dated run per model/agent. Different language task sets are not averaged together.", "zh": "Rust Verified 子集；采用榜单已验证提交，各模型与 Agent 使用最新日期记录。不同语言任务集不混合平均。" }, "source": { "name": "ByteDance Seed", "benchmark_creator": "ByteDance Seed", "url": "https://multi-swe-bench.github.io/", "data_url": "https://raw.githubusercontent.com/multi-swe-bench/experiments/refs/heads/dist/leaderboard.json", "sha256": "7ac5c3356b384b6eabe211b331ca72e7766832eca7a92ae954de09f893cacd72", "retrieved_at": "2026-11-02T03:36:07.507718+00:00", "selector": "Rust/Verified: verified=true, resolvedRate; latest date per model+agent", "kind": "official_submissions", "status": "current_snapshot" }, "results": [ { "id": "14778b809507673ee6ea", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.1589958158995816, "raw_score": 0.1589958158995816, "raw_scale": 1, "source_record": "rust/verified/20250329_MopenHands_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MopenHands_Claude-3.7-Sonnet" }, { "id": "f960b971663a90d8acde", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MopenHands", "score": 0.14644351464435146, "raw_score": 0.14644351464435146, "raw_scale": 1, "source_record": "rust/verified/20250426_MopenHands_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250426_MopenHands_Gemini-2.5-Pro" }, { "id": "5862c114f9438f8eb1f2", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.12133891213389121, "raw_score": 0.12133891213389121, "raw_scale": 1, "source_record": "rust/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)" }, { "id": "713a37aabf4ddf6e9408", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MSWE-agent", "score": 0.100418410041841, "raw_score": 0.100418410041841, "raw_scale": 1, "source_record": "rust/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250426_MSWE-agent_Gemini-2.5-Pro" }, { "id": "2afa067a5a079e7d6dbd", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.0794979079497908, "raw_score": 0.0794979079497908, "raw_scale": 1, "source_record": "rust/verified/20250329_MagentLess_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MagentLess_OpenAI-o3-mini-high" }, { "id": "d0cdb9a76e811bd3fa02", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MagentLess", "score": 0.07112970711297072, "raw_score": 0.07112970711297072, "raw_scale": 1, "source_record": "rust/verified/20250426_MagentLess_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250426_MagentLess_Doubao-1.5-thinking" }, { "id": "fff46d8e2028af839db4", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.07112970711297072, "raw_score": 0.07112970711297072, "raw_scale": 1, "source_record": "rust/verified/20250329_MagentLess_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MagentLess_OpenAI-o1" }, { "id": "7941ff38630c96d2caec", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.06694560669456066, "raw_score": 0.06694560669456066, "raw_scale": 1, "source_record": "rust/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)" }, { "id": "b6d6cdd4f53c3178b3a7", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.06694560669456066, "raw_score": 0.06694560669456066, "raw_scale": 1, "source_record": "rust/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MSWE-agent_Claude-3.7-Sonnet" }, { "id": "27c38e8a2d71e6baa40b", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.06694560669456066, "raw_score": 0.06694560669456066, "raw_scale": 1, "source_record": "rust/verified/20250329_MagentLess_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MagentLess_DeepSeek-R1" }, { "id": "b900cbac44742248ff9b", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MagentLess", "score": 0.06276150627615062, "raw_score": 0.06276150627615062, "raw_scale": 1, "source_record": "rust/verified/20250426_MagentLess_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250426_MagentLess_Llama-4-Maverick" }, { "id": "c5f189dc636c2c0fec02", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.058577405857740586, "raw_score": 0.058577405857740586, "raw_scale": 1, "source_record": "rust/verified/20250329_MSWE-agent_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MSWE-agent_DeepSeek-V3" }, { "id": "30391339ad8e62e329d3", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.058577405857740586, "raw_score": 0.058577405857740586, "raw_scale": 1, "source_record": "rust/verified/20250329_MagentLess_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MagentLess_GPT-4o-1120" }, { "id": "dfa1592f179e1d0f16fa", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.05439330543933055, "raw_score": 0.05439330543933055, "raw_scale": 1, "source_record": "rust/verified/20250329_MagentLess_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MagentLess_Claude-3.7-Sonnet" }, { "id": "0ef9591bc620d5ad50cf", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MagentLess", "score": 0.05439330543933055, "raw_score": 0.05439330543933055, "raw_scale": 1, "source_record": "rust/verified/20250426_MagentLess_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250426_MagentLess_Gemini-2.5-Pro" }, { "id": "204ecf377a7ba9dd29a9", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.0502092050209205, "raw_score": 0.0502092050209205, "raw_scale": 1, "source_record": "rust/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)" }, { "id": "bfe1771bde1977208ca2", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.0502092050209205, "raw_score": 0.0502092050209205, "raw_scale": 1, "source_record": "rust/verified/20250329_MagentLess_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MagentLess_DeepSeek-V3" }, { "id": "7d2ba01ff8ec3c7f3212", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MopenHands", "score": 0.0502092050209205, "raw_score": 0.0502092050209205, "raw_scale": 1, "source_record": "rust/verified/20250426_MopenHands_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250426_MopenHands_Llama-4-Maverick" }, { "id": "b64080072c5409292cae", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.0502092050209205, "raw_score": 0.0502092050209205, "raw_scale": 1, "source_record": "rust/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MSWE-agent_OpenAI-o3-mini-high" }, { "id": "84b217f9890a71b508d0", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.0502092050209205, "raw_score": 0.0502092050209205, "raw_scale": 1, "source_record": "rust/verified/20250329_MopenHands_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MopenHands_OpenAI-o3-mini-high" }, { "id": "92aeebd8229b6ed99e65", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.04602510460251046, "raw_score": 0.04602510460251046, "raw_scale": 1, "source_record": "rust/verified/20250329_MopenHands_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MopenHands_DeepSeek-R1" }, { "id": "b3bd6c1114e2abd44fb1", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.04602510460251046, "raw_score": 0.04602510460251046, "raw_scale": 1, "source_record": "rust/verified/20250329_MopenHands_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MopenHands_DeepSeek-V3" }, { "id": "19c13a56b5dda4cf2ede", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MagentLess", "score": 0.04184100418410042, "raw_score": 0.04184100418410042, "raw_scale": 1, "source_record": "rust/verified/20250329_MagentLess_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MagentLess_Doubao-1.5-pro" }, { "id": "5951e9ddb539408b6be7", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.04184100418410042, "raw_score": 0.04184100418410042, "raw_scale": 1, "source_record": "rust/verified/20250329_MSWE-agent_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MSWE-agent_OpenAI-o1" }, { "id": "8832e8ee2b823e4a82ea", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.03347280334728033, "raw_score": 0.03347280334728033, "raw_scale": 1, "source_record": "rust/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250426_MSWE-agent_Doubao-1.5-thinking" }, { "id": "3649c3846e95ceaa6040", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MopenHands", "score": 0.03347280334728033, "raw_score": 0.03347280334728033, "raw_scale": 1, "source_record": "rust/verified/20250426_MopenHands_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250426_MopenHands_Doubao-1.5-thinking" }, { "id": "b90b6eed1db38bf9a5a9", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.03347280334728033, "raw_score": 0.03347280334728033, "raw_scale": 1, "source_record": "rust/verified/20250329_MopenHands_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MopenHands_GPT-4o-1120" }, { "id": "8ff149bc230cad22f6dc", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MSWE-agent", "score": 0.02510460251046025, "raw_score": 0.02510460251046025, "raw_scale": 1, "source_record": "rust/verified/20250426_MSWE-agent_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250426_MSWE-agent_Llama-4-Maverick" }, { "id": "b6cedc0fb2fe3d5c695f", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.02510460251046025, "raw_score": 0.02510460251046025, "raw_scale": 1, "source_record": "rust/verified/20250329_MopenHands_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MopenHands_OpenAI-o1" }, { "id": "6dabd22c7a6b66aa5837", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MagentLess", "score": 0.02510460251046025, "raw_score": 0.02510460251046025, "raw_scale": 1, "source_record": "rust/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MagentLess_Qwen2.5-72B-Instruct" }, { "id": "fbf05264598cd7f2c3a5", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.02092050209205021, "raw_score": 0.02092050209205021, "raw_scale": 1, "source_record": "rust/verified/20250329_MSWE-agent_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MSWE-agent_DeepSeek-R1" }, { "id": "eded24f3c484b21a8956", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.02092050209205021, "raw_score": 0.02092050209205021, "raw_scale": 1, "source_record": "rust/verified/20250329_MSWE-agent_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MSWE-agent_GPT-4o-1120" }, { "id": "707e2b9b7e713895202f", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.016736401673640166, "raw_score": 0.016736401673640166, "raw_scale": 1, "source_record": "rust/verified/20250329_MSWE-agent_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MSWE-agent_Doubao-1.5-pro" }, { "id": "283716794e58f3bc259a", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MopenHands", "score": 0.016736401673640166, "raw_score": 0.016736401673640166, "raw_scale": 1, "source_record": "rust/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MopenHands_Qwen2.5-72B-Instruct" }, { "id": "b998a814bd6118006fee", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MopenHands", "score": 0.008368200836820083, "raw_score": 0.008368200836820083, "raw_scale": 1, "source_record": "rust/verified/20250329_MopenHands_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MopenHands_Doubao-1.5-pro" }, { "id": "51b7db6bcd773d1aac8f", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MSWE-agent", "score": 0.0041841004184100415, "raw_score": 0.0041841004184100415, "raw_scale": 1, "source_record": "rust/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/rust/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct" } ] }, { "id": "multi-swe-bench-c", "name": "Multi-SWE-bench (C)", "group": "编程", "unit": "%", "metric": "Resolved rate", "protocol": { "en": "C Verified subset; verified leaderboard submissions, latest dated run per model/agent. Different language task sets are not averaged together.", "zh": "C Verified 子集；采用榜单已验证提交，各模型与 Agent 使用最新日期记录。不同语言任务集不混合平均。" }, "source": { "name": "ByteDance Seed", "benchmark_creator": "ByteDance Seed", "url": "https://multi-swe-bench.github.io/", "data_url": "https://raw.githubusercontent.com/multi-swe-bench/experiments/refs/heads/dist/leaderboard.json", "sha256": "7ac5c3356b384b6eabe211b331ca72e7766832eca7a92ae954de09f893cacd72", "retrieved_at": "2026-11-02T03:36:07.507718+00:00", "selector": "C/Verified: verified=true, resolvedRate; latest date per model+agent", "kind": "official_submissions", "status": "current_snapshot" }, "results": [ { "id": "2d2adcf70da356a47be3", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MSWE-agent", "score": 0.09375, "raw_score": 0.09375, "raw_scale": 1, "source_record": "c/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250426_MSWE-agent_Gemini-2.5-Pro" }, { "id": "a93122ef54b3da2eda58", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MagentLess", "score": 0.09375, "raw_score": 0.09375, "raw_scale": 1, "source_record": "c/verified/20250426_MagentLess_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250426_MagentLess_Gemini-2.5-Pro" }, { "id": "a05f4ac188837f563f71", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.0859375, "raw_score": 0.0859375, "raw_scale": 1, "source_record": "c/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MSWE-agent_Claude-3.7-Sonnet" }, { "id": "0754d0b6a6f022cb0fee", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.0859375, "raw_score": 0.0859375, "raw_scale": 1, "source_record": "c/verified/20250329_MopenHands_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MopenHands_Claude-3.7-Sonnet" }, { "id": "335eea03d5295e1bfe23", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.0625, "raw_score": 0.0625, "raw_scale": 1, "source_record": "c/verified/20250329_MagentLess_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MagentLess_Claude-3.7-Sonnet" }, { "id": "5d1ea701f6fa4cefca08", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MopenHands", "score": 0.0546875, "raw_score": 0.0546875, "raw_scale": 1, "source_record": "c/verified/20250426_MopenHands_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250426_MopenHands_Gemini-2.5-Pro" }, { "id": "9521a54f70e63e303320", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.046875, "raw_score": 0.046875, "raw_scale": 1, "source_record": "c/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)" }, { "id": "cb524a17d5110c6ed818", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MagentLess", "score": 0.046875, "raw_score": 0.046875, "raw_scale": 1, "source_record": "c/verified/20250426_MagentLess_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250426_MagentLess_Doubao-1.5-thinking" }, { "id": "2051e2d0afc54c8af337", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MopenHands", "score": 0.0390625, "raw_score": 0.0390625, "raw_scale": 1, "source_record": "c/verified/20250426_MopenHands_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250426_MopenHands_Doubao-1.5-thinking" }, { "id": "09328c738c3bfb1c46df", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MagentLess", "score": 0.0390625, "raw_score": 0.0390625, "raw_scale": 1, "source_record": "c/verified/20250426_MagentLess_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250426_MagentLess_Llama-4-Maverick" }, { "id": "50b032350bfd17a57a52", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.0390625, "raw_score": 0.0390625, "raw_scale": 1, "source_record": "c/verified/20250329_MSWE-agent_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MSWE-agent_OpenAI-o1" }, { "id": "81d20129528dd64cf32e", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.03125, "raw_score": 0.03125, "raw_scale": 1, "source_record": "c/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)" }, { "id": "6d20d66dc9c62c65ba06", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.03125, "raw_score": 0.03125, "raw_scale": 1, "source_record": "c/verified/20250329_MagentLess_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MagentLess_DeepSeek-V3" }, { "id": "c693907633cca431c838", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.03125, "raw_score": 0.03125, "raw_scale": 1, "source_record": "c/verified/20250329_MopenHands_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MopenHands_DeepSeek-V3" }, { "id": "0ae1abf2b2a5a25d3f04", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MopenHands", "score": 0.03125, "raw_score": 0.03125, "raw_scale": 1, "source_record": "c/verified/20250426_MopenHands_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250426_MopenHands_Llama-4-Maverick" }, { "id": "e7d71cd1171f12eb5f62", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.03125, "raw_score": 0.03125, "raw_scale": 1, "source_record": "c/verified/20250329_MagentLess_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MagentLess_OpenAI-o1" }, { "id": "26de5444680de994fc1d", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.03125, "raw_score": 0.03125, "raw_scale": 1, "source_record": "c/verified/20250329_MopenHands_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MopenHands_OpenAI-o1" }, { "id": "5f72491677204d0254b2", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.0234375, "raw_score": 0.0234375, "raw_scale": 1, "source_record": "c/verified/20250329_MopenHands_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MopenHands_DeepSeek-R1" }, { "id": "c75ce985a30df0dbfb34", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.0234375, "raw_score": 0.0234375, "raw_scale": 1, "source_record": "c/verified/20250329_MSWE-agent_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MSWE-agent_DeepSeek-V3" }, { "id": "e3b3db5082a4c05b9693", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.0234375, "raw_score": 0.0234375, "raw_scale": 1, "source_record": "c/verified/20250329_MSWE-agent_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MSWE-agent_Doubao-1.5-pro" }, { "id": "4bc0528a70ef62ff7292", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.0234375, "raw_score": 0.0234375, "raw_scale": 1, "source_record": "c/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MSWE-agent_OpenAI-o3-mini-high" }, { "id": "80dc2eb78228b10d30c7", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.0234375, "raw_score": 0.0234375, "raw_scale": 1, "source_record": "c/verified/20250329_MagentLess_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MagentLess_OpenAI-o3-mini-high" }, { "id": "74a98062b3df1659a2af", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.015625, "raw_score": 0.015625, "raw_scale": 1, "source_record": "c/verified/20250329_MSWE-agent_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MSWE-agent_GPT-4o-1120" }, { "id": "98667a2287e56af981b1", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.015625, "raw_score": 0.015625, "raw_scale": 1, "source_record": "c/verified/20250329_MagentLess_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MagentLess_GPT-4o-1120" }, { "id": "5c505c74d09c98c1c185", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.015625, "raw_score": 0.015625, "raw_scale": 1, "source_record": "c/verified/20250329_MopenHands_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MopenHands_OpenAI-o3-mini-high" }, { "id": "2f6435341a8c24ed1fe4", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MSWE-agent", "score": 0.015625, "raw_score": 0.015625, "raw_scale": 1, "source_record": "c/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct" }, { "id": "beb2777b0b623cc33376", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MagentLess", "score": 0.015625, "raw_score": 0.015625, "raw_scale": 1, "source_record": "c/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MagentLess_Qwen2.5-72B-Instruct" }, { "id": "fc6e9a6bfa580f2fd846", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.0078125, "raw_score": 0.0078125, "raw_scale": 1, "source_record": "c/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)" }, { "id": "67bc1c922b99ecbde91c", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.0078125, "raw_score": 0.0078125, "raw_scale": 1, "source_record": "c/verified/20250329_MSWE-agent_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MSWE-agent_DeepSeek-R1" }, { "id": "a617eb25234948e23f3a", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.0078125, "raw_score": 0.0078125, "raw_scale": 1, "source_record": "c/verified/20250329_MagentLess_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MagentLess_DeepSeek-R1" }, { "id": "502b1c47f7e61c32b857", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MagentLess", "score": 0.0078125, "raw_score": 0.0078125, "raw_scale": 1, "source_record": "c/verified/20250329_MagentLess_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MagentLess_Doubao-1.5-pro" }, { "id": "6e3c0759797b5c30e5d4", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.0078125, "raw_score": 0.0078125, "raw_scale": 1, "source_record": "c/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250426_MSWE-agent_Doubao-1.5-thinking" }, { "id": "e09c04007308b40fc615", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MopenHands", "score": 0.0078125, "raw_score": 0.0078125, "raw_scale": 1, "source_record": "c/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MopenHands_Qwen2.5-72B-Instruct" }, { "id": "9e78940dd9eed647913a", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MopenHands", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "c/verified/20250329_MopenHands_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MopenHands_Doubao-1.5-pro" }, { "id": "eb5a104b9da54a0d6685", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "c/verified/20250329_MopenHands_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250329_MopenHands_GPT-4o-1120" }, { "id": "d211fc1669ba20052438", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MSWE-agent", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "c/verified/20250426_MSWE-agent_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c/verified/20250426_MSWE-agent_Llama-4-Maverick" } ] }, { "id": "multi-swe-bench-cpp", "name": "Multi-SWE-bench (C++)", "group": "编程", "unit": "%", "metric": "Resolved rate", "protocol": { "en": "C++ Verified subset; verified leaderboard submissions, latest dated run per model/agent. Different language task sets are not averaged together.", "zh": "C++ Verified 子集；采用榜单已验证提交，各模型与 Agent 使用最新日期记录。不同语言任务集不混合平均。" }, "source": { "name": "ByteDance Seed", "benchmark_creator": "ByteDance Seed", "url": "https://multi-swe-bench.github.io/", "data_url": "https://raw.githubusercontent.com/multi-swe-bench/experiments/refs/heads/dist/leaderboard.json", "sha256": "7ac5c3356b384b6eabe211b331ca72e7766832eca7a92ae954de09f893cacd72", "retrieved_at": "2026-11-02T03:36:07.507718+00:00", "selector": "C++/Verified: verified=true, resolvedRate; latest date per model+agent", "kind": "official_submissions", "status": "current_snapshot" }, "results": [ { "id": "36fa5a03ed0011d39a1c", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.14728682170542637, "raw_score": 0.14728682170542637, "raw_scale": 1, "source_record": "c++/verified/20250329_MopenHands_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MopenHands_Claude-3.7-Sonnet" }, { "id": "73a7c721e9e9e6074115", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MopenHands", "score": 0.12403100775193798, "raw_score": 0.12403100775193798, "raw_scale": 1, "source_record": "c++/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MopenHands_Claude-3.5-Sonnet(Oct)" }, { "id": "293fc2c6b044908b5f7b", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.11627906976744186, "raw_score": 0.11627906976744186, "raw_scale": 1, "source_record": "c++/verified/20250329_MSWE-agent_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MSWE-agent_Claude-3.7-Sonnet" }, { "id": "509ef23bcdf183531769", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MopenHands", "score": 0.09302325581395349, "raw_score": 0.09302325581395349, "raw_scale": 1, "source_record": "c++/verified/20250426_MopenHands_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250426_MopenHands_Gemini-2.5-Pro" }, { "id": "4344c40a3c834abd91ac", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MSWE-agent", "score": 0.08527131782945736, "raw_score": 0.08527131782945736, "raw_scale": 1, "source_record": "c++/verified/20250426_MSWE-agent_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250426_MSWE-agent_Gemini-2.5-Pro" }, { "id": "5e268bd41f389714d6ef", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.07751937984496124, "raw_score": 0.07751937984496124, "raw_scale": 1, "source_record": "c++/verified/20250329_MSWE-agent_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MSWE-agent_DeepSeek-V3" }, { "id": "9ec592269f14a6844478", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.07751937984496124, "raw_score": 0.07751937984496124, "raw_scale": 1, "source_record": "c++/verified/20250329_MopenHands_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MopenHands_DeepSeek-V3" }, { "id": "2e14553cb06e0e1962f6", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MSWE-agent", "score": 0.06976744186046512, "raw_score": 0.06976744186046512, "raw_scale": 1, "source_record": "c++/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MSWE-agent_Claude-3.5-Sonnet(Oct)" }, { "id": "9f298a442b21ce359437", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.06976744186046512, "raw_score": 0.06976744186046512, "raw_scale": 1, "source_record": "c++/verified/20250329_MagentLess_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MagentLess_GPT-4o-1120" }, { "id": "43e04aa1455c7195bd96", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MSWE-agent", "score": 0.06976744186046512, "raw_score": 0.06976744186046512, "raw_scale": 1, "source_record": "c++/verified/20250426_MSWE-agent_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250426_MSWE-agent_Llama-4-Maverick" }, { "id": "ec3d4e40f17adf8ec29d", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.06976744186046512, "raw_score": 0.06976744186046512, "raw_scale": 1, "source_record": "c++/verified/20250329_MopenHands_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MopenHands_OpenAI-o3-mini-high" }, { "id": "f7ef3b572b6414875858", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MSWE-agent", "score": 0.06201550387596899, "raw_score": 0.06201550387596899, "raw_scale": 1, "source_record": "c++/verified/20250329_MSWE-agent_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MSWE-agent_DeepSeek-R1" }, { "id": "ef0df7d40c5f34fd4222", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.06201550387596899, "raw_score": 0.06201550387596899, "raw_scale": 1, "source_record": "c++/verified/20250329_MSWE-agent_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MSWE-agent_Doubao-1.5-pro" }, { "id": "bd73cb857e5a8814dd97", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MopenHands", "score": 0.05426356589147287, "raw_score": 0.05426356589147287, "raw_scale": 1, "source_record": "c++/verified/20250426_MopenHands_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250426_MopenHands_Doubao-1.5-thinking" }, { "id": "f4b36456232103a56538", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MagentLess", "score": 0.05426356589147287, "raw_score": 0.05426356589147287, "raw_scale": 1, "source_record": "c++/verified/20250426_MagentLess_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250426_MagentLess_Llama-4-Maverick" }, { "id": "d6db48498bb3a90fceb7", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.05426356589147287, "raw_score": 0.05426356589147287, "raw_scale": 1, "source_record": "c++/verified/20250329_MagentLess_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MagentLess_OpenAI-o1" }, { "id": "d115c7e5aa2f2fc11de4", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.05426356589147287, "raw_score": 0.05426356589147287, "raw_scale": 1, "source_record": "c++/verified/20250329_MSWE-agent_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MSWE-agent_OpenAI-o3-mini-high" }, { "id": "303fe0ade99372d78138", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MopenHands", "score": 0.046511627906976744, "raw_score": 0.046511627906976744, "raw_scale": 1, "source_record": "c++/verified/20250329_MopenHands_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MopenHands_DeepSeek-R1" }, { "id": "56d4befd1f03f123d2cb", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MSWE-agent", "score": 0.046511627906976744, "raw_score": 0.046511627906976744, "raw_scale": 1, "source_record": "c++/verified/20250426_MSWE-agent_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250426_MSWE-agent_Doubao-1.5-thinking" }, { "id": "2b0219559bb8c16b9089", "model": "Claude-3.5-Sonnet(Oct)", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.03875968992248062, "raw_score": 0.03875968992248062, "raw_scale": 1, "source_record": "c++/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MagentLess_Claude-3.5-Sonnet(Oct)" }, { "id": "cb3f93489a0c18f7880c", "model": "Doubao-1.5-thinking", "organization": "", "configuration": "Agent: MagentLess", "score": 0.03875968992248062, "raw_score": 0.03875968992248062, "raw_scale": 1, "source_record": "c++/verified/20250426_MagentLess_Doubao-1.5-thinking", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250426_MagentLess_Doubao-1.5-thinking" }, { "id": "2e2a955195b48ad74f81", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.03875968992248062, "raw_score": 0.03875968992248062, "raw_scale": 1, "source_record": "c++/verified/20250329_MopenHands_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MopenHands_GPT-4o-1120" }, { "id": "4c39f2a84c5100d2be33", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.03875968992248062, "raw_score": 0.03875968992248062, "raw_scale": 1, "source_record": "c++/verified/20250329_MSWE-agent_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MSWE-agent_OpenAI-o1" }, { "id": "f97eb02cd15ebe360103", "model": "OpenAI-o1", "organization": "OpenAI", "configuration": "Agent: MopenHands", "score": 0.03875968992248062, "raw_score": 0.03875968992248062, "raw_scale": 1, "source_record": "c++/verified/20250329_MopenHands_OpenAI-o1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MopenHands_OpenAI-o1" }, { "id": "e60cd40a735ed1778ae7", "model": "Claude-3.7-Sonnet", "organization": "Anthropic", "configuration": "Agent: MagentLess", "score": 0.031007751937984496, "raw_score": 0.031007751937984496, "raw_scale": 1, "source_record": "c++/verified/20250329_MagentLess_Claude-3.7-Sonnet", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MagentLess_Claude-3.7-Sonnet" }, { "id": "4499798bf359732f01ad", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.031007751937984496, "raw_score": 0.031007751937984496, "raw_scale": 1, "source_record": "c++/verified/20250329_MagentLess_DeepSeek-R1", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MagentLess_DeepSeek-R1" }, { "id": "9c0ce02646ef1bc15e09", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MopenHands", "score": 0.031007751937984496, "raw_score": 0.031007751937984496, "raw_scale": 1, "source_record": "c++/verified/20250329_MopenHands_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MopenHands_Doubao-1.5-pro" }, { "id": "94d40bc6f712f00dabbe", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Agent: MopenHands", "score": 0.031007751937984496, "raw_score": 0.031007751937984496, "raw_scale": 1, "source_record": "c++/verified/20250426_MopenHands_Llama-4-Maverick", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250426_MopenHands_Llama-4-Maverick" }, { "id": "8e65fc34d42220d1b3dd", "model": "GPT-4o-1120", "organization": "OpenAI", "configuration": "Agent: MSWE-agent", "score": 0.023255813953488372, "raw_score": 0.023255813953488372, "raw_scale": 1, "source_record": "c++/verified/20250329_MSWE-agent_GPT-4o-1120", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MSWE-agent_GPT-4o-1120" }, { "id": "45531bdb307cd2210f64", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: MagentLess", "score": 0.023255813953488372, "raw_score": 0.023255813953488372, "raw_scale": 1, "source_record": "c++/verified/20250426_MagentLess_Gemini-2.5-Pro", "source_date": "2025-04-26", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250426_MagentLess_Gemini-2.5-Pro" }, { "id": "9d7c640c5ca5732cc6f4", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MopenHands", "score": 0.023255813953488372, "raw_score": 0.023255813953488372, "raw_scale": 1, "source_record": "c++/verified/20250329_MopenHands_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MopenHands_Qwen2.5-72B-Instruct" }, { "id": "a70e7930e9d1db301a39", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Agent: MagentLess", "score": 0.015503875968992248, "raw_score": 0.015503875968992248, "raw_scale": 1, "source_record": "c++/verified/20250329_MagentLess_DeepSeek-V3", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MagentLess_DeepSeek-V3" }, { "id": "25fd88961f66d8b4b73a", "model": "OpenAI-o3-mini-high", "organization": "OpenAI", "configuration": "Agent: MagentLess", "score": 0.015503875968992248, "raw_score": 0.015503875968992248, "raw_scale": 1, "source_record": "c++/verified/20250329_MagentLess_OpenAI-o3-mini-high", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MagentLess_OpenAI-o3-mini-high" }, { "id": "724fd7f8ec9b2e03a9b3", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MagentLess", "score": 0.007751937984496124, "raw_score": 0.007751937984496124, "raw_scale": 1, "source_record": "c++/verified/20250329_MagentLess_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MagentLess_Qwen2.5-72B-Instruct" }, { "id": "3d0092a27c110bb80e89", "model": "Doubao-1.5-pro", "organization": "", "configuration": "Agent: MagentLess", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "c++/verified/20250329_MagentLess_Doubao-1.5-pro", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MagentLess_Doubao-1.5-pro" }, { "id": "e126d67e26c86e9ce657", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Agent: MSWE-agent", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "c++/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct", "source_date": "2025-03-29", "report_url": "https://github.com/multi-swe-bench/experiments/tree/main/evaluation/c++/verified/20250329_MSWE-agent_Qwen2.5-72B-Instruct" } ] }, { "id": "explorationbench-aliencode", "name": "ExplorationBench (AlienCode)", "group": "数学与科学", "unit": "%", "metric": "Mean@3 held-out accuracy", "protocol": { "en": "AlienCode: held-out accuracy after four exploration rounds (M4), averaged across three independent trajectories; each task is answered three times. Highest reasoning setting. Not the best@3 column.", "zh": "AlienCode：四轮探索后（M4）的保留题正确率，三条独立轨迹取平均，每题回答三次；使用最高推理设置，不读取 best@3 列。" }, "source": { "name": "Tencent Hunyuan", "benchmark_creator": "Tencent Hunyuan", "url": "https://explorationbench.com/", "data_url": "https://raw.githubusercontent.com/Tencent-Hunyuan/ExplorationBench/main/README.md", "sha256": "1b45c56f72ef61a8f7335b0a110776563b9ae989bc2aad0896736b8affb8b8b3", "retrieved_at": "2026-11-02T03:34:36.375193+00:00", "selector": "AlienCode table: Mean@3 column", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "581f81977e7f921ecee4", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.795, "raw_score": 79.5, "raw_scale": 100, "source_record": "GPT-5.6 Sol|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "b5f31e3c27ef635274f2", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.735, "raw_score": 73.5, "raw_scale": 100, "source_record": "Claude Opus 5|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "96eaac8b804e950a1c95", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.64, "raw_score": 64.0, "raw_scale": 100, "source_record": "Qwen3.8-Max|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "c18180b7eba63a7aa726", "model": "Grok 4.6", "organization": "xAI", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.532, "raw_score": 53.2, "raw_scale": 100, "source_record": "Grok 4.6|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "cf820734978bb511e47a", "model": "DeepSeek-V4.1-Flash", "organization": "DeepSeek", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.46299999999999997, "raw_score": 46.3, "raw_scale": 100, "source_record": "DeepSeek-V4.1-Flash|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "c4e310c8fffc7998a158", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.397, "raw_score": 39.7, "raw_scale": 100, "source_record": "Gemini 3.8 Flash|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "c6ab37fec7fbc93f9a70", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.387, "raw_score": 38.7, "raw_scale": 100, "source_record": "Kimi K3|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "420999004948d66f941e", "model": "Hy4 preview", "organization": "", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.302, "raw_score": 30.2, "raw_scale": 100, "source_record": "Hy4 preview|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "315957bc71a6fcc3133d", "model": "Seed2.1 Pro", "organization": "", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.248, "raw_score": 24.8, "raw_scale": 100, "source_record": "Seed2.1 Pro|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "08a3cb2df22cb3f3c6a3", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.073, "raw_score": 7.3, "raw_scale": 100, "source_record": "DeepSeek-V4-Pro|Highest reasoning; M4; Mean@3 independent trajectories" } ] }, { "id": "explorationbench-alienlogic", "name": "ExplorationBench (AlienLogic)", "group": "数学与科学", "unit": "%", "metric": "Mean@3 held-out accuracy", "protocol": { "en": "AlienLogic: held-out accuracy after four exploration rounds (M4), averaged across three independent trajectories; each task is answered three times. Highest reasoning setting. Not the best@3 column.", "zh": "AlienLogic：四轮探索后（M4）的保留题正确率，三条独立轨迹取平均，每题回答三次；使用最高推理设置，不读取 best@3 列。" }, "source": { "name": "Tencent Hunyuan", "benchmark_creator": "Tencent Hunyuan", "url": "https://explorationbench.com/", "data_url": "https://raw.githubusercontent.com/Tencent-Hunyuan/ExplorationBench/main/README.md", "sha256": "1b45c56f72ef61a8f7335b0a110776563b9ae989bc2aad0896736b8affb8b8b3", "retrieved_at": "2026-11-02T03:34:36.375193+00:00", "selector": "AlienLogic table: Mean@3 column", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "b5f31e3c27ef635274f2", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.759, "raw_score": 75.9, "raw_scale": 100, "source_record": "Claude Opus 5|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "581f81977e7f921ecee4", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.7509999999999999, "raw_score": 75.1, "raw_scale": 100, "source_record": "GPT-5.6 Sol|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "96eaac8b804e950a1c95", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.743, "raw_score": 74.3, "raw_scale": 100, "source_record": "Qwen3.8-Max|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "c18180b7eba63a7aa726", "model": "Grok 4.6", "organization": "xAI", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.721, "raw_score": 72.1, "raw_scale": 100, "source_record": "Grok 4.6|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "c6ab37fec7fbc93f9a70", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.6940000000000001, "raw_score": 69.4, "raw_scale": 100, "source_record": "Kimi K3|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "420999004948d66f941e", "model": "Hy4 preview", "organization": "", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.6920000000000001, "raw_score": 69.2, "raw_scale": 100, "source_record": "Hy4 preview|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "08a3cb2df22cb3f3c6a3", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.6629999999999999, "raw_score": 66.3, "raw_scale": 100, "source_record": "DeepSeek-V4-Pro|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "cf820734978bb511e47a", "model": "DeepSeek-V4.1-Flash", "organization": "DeepSeek", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.606, "raw_score": 60.6, "raw_scale": 100, "source_record": "DeepSeek-V4.1-Flash|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "315957bc71a6fcc3133d", "model": "Seed2.1 Pro", "organization": "", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.598, "raw_score": 59.8, "raw_scale": 100, "source_record": "Seed2.1 Pro|Highest reasoning; M4; Mean@3 independent trajectories" }, { "id": "c4e310c8fffc7998a158", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Highest reasoning; M4; Mean@3 independent trajectories", "score": 0.59, "raw_score": 59.0, "raw_scale": 100, "source_record": "Gemini 3.8 Flash|Highest reasoning; M4; Mean@3 independent trajectories" } ] }, { "id": "google-facts-v1", "name": "FACTS Benchmark Suite (v1)", "group": "知识与推理", "unit": "score", "metric": "FACTS composite score", "protocol": { "en": "Official Kaggle board, benchmark version 1; current mapped task version 108. Average across FACTS components, shown on 0–100 and excluded from saturation bins. Requires all four currently mapped component versions; older component mixtures are omitted.", "zh": "官方 Kaggle 榜单，评测版本 1，当前映射的任务版本 108。FACTS 分项平均，显示为 0–100 综合分，不进入饱和度分组；须具备四个当前分项版本的成绩，排除旧版分项混用。" }, "source": { "name": "Google DeepMind / Kaggle", "benchmark_creator": "Google DeepMind", "url": "https://www.kaggle.com/benchmarks/google/facts/leaderboard", "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmarkLeaderboard", "sha256": "d4ffe5aff17b394969aefd9bf394468ed2d032a2d014d50e1a397d447e9f1c0f", "retrieved_at": "2026-11-02T03:38:07.967518+00:00", "selector": "GetBenchmarkLeaderboard versionIdSelector.id=6; rows.results[taskVersionId=108].numericResult.value", "kind": "official_leaderboard", "status": "current_snapshot", "request_body": { "versionIdentifier": { "versionIdSelector": { "id": 6 } } } }, "results": [ { "id": "734d0759cdb4e0d0a35e", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 75.39661520133869, "raw_score": 0.7539661520133869, "raw_scale": 0.01, "source_record": "170", "source_date": "2026-10-26T10:36:05.346607Z", "source_model_id": "openai/gpt-6-astra" }, { "id": "2811745d7b8d8874f6e6", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 74.60844650396427, "raw_score": 0.7460844650396428, "raw_scale": 0.01, "source_record": "186", "source_date": "2026-10-26T10:37:32.627328Z", "source_model_id": "openai/gpt-6.1-sol" }, { "id": "80c3cd40fa35f9088b87", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 68.47424679788553, "raw_score": 0.6847424679788553, "raw_scale": 0.01, "source_record": "168", "source_date": "2026-10-25T18:20:56.269Z", "source_model_id": "google/gemini-3.8-flash" }, { "id": "3068430da9e4b7a67418", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 64.0159443872485, "raw_score": 0.6401594438724849, "raw_scale": 0.01, "source_record": "179", "source_date": "2026-10-26T10:37:41.492282Z", "source_model_id": "anthropic/claude-opus-5-5@default" }, { "id": "580811fa95269f3ecd4f", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 62.34009939071006, "raw_score": 0.6234009939071006, "raw_scale": 0.01, "source_record": "181", "source_date": "2026-10-27T17:23:21.569825Z", "source_model_id": "openai/gpt-6-luna" } ], "score_max": 100 }, { "id": "google-facts-grounding-v3", "name": "FACTS Grounding (v3)", "group": "知识与推理", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Official Kaggle board, benchmark version 3; current mapped task version 1109072. Published fractional accuracy; missing model results are omitted.", "zh": "官方 Kaggle 榜单，评测版本 3，当前映射的任务版本 1109072。官方正确率；未公布成绩的模型不填分。" }, "source": { "name": "Google DeepMind / Kaggle", "benchmark_creator": "Google DeepMind", "url": "https://www.kaggle.com/benchmarks/google/facts-grounding/leaderboard", "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmarkLeaderboard", "sha256": "d4ffe5aff17b394969aefd9bf394468ed2d032a2d014d50e1a397d447e9f1c0f", "retrieved_at": "2026-11-02T05:58:08.945813+00:00", "selector": "GetBenchmarkLeaderboard versionIdSelector.id=6 or 8542; rows.results[taskVersionId=1109072].numericResult.value", "kind": "official_leaderboard", "status": "current_snapshot", "request_body": { "versionIdentifier": { "versionIdSelector": { "id": 6 } } }, "artifacts": [ { "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmarkLeaderboard", "sha256": "9ca7fb1e981eb95653eac102ba59d3a0b486ecfba0a7a84af26ba76db7812339", "retrieved_at": "2026-11-02T05:58:34.726578+00:00", "request_body": { "versionIdentifier": { "versionIdSelector": { "id": 8542 } } } } ] }, "results": [ { "id": "734d0759cdb4e0d0a35e", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8011661807580175, "raw_score": 0.8011661807580175, "raw_scale": 1, "source_record": "170", "source_date": "2026-10-24T01:19:11.798251Z", "source_model_id": "openai/gpt-6-astra", "source_version_ids": [ 6, 8542 ], "source_task_version_id": 1109072 }, { "id": "2811745d7b8d8874f6e6", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.780466472303207, "raw_score": 0.780466472303207, "raw_scale": 1, "source_record": "186", "source_date": "2026-10-24T01:20:51.600302Z", "source_model_id": "openai/gpt-6.1-sol", "source_version_ids": [ 6, 8542 ], "source_task_version_id": 1109072 }, { "id": "8d27ba37c5d810106b55", "model": "Gemma 4 26B A4B", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.7778425655976676, "raw_score": 0.7778425655976676, "raw_scale": 1, "source_record": "139", "source_date": "2026-10-24T02:18:40.696906Z", "source_model_id": "google/gemma-4-26b-a4b", "source_version_ids": [ 6, 8542 ], "source_task_version_id": 1109072 }, { "id": "580811fa95269f3ecd4f", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.7670553935860058, "raw_score": 0.7670553935860058, "raw_scale": 1, "source_record": "181", "source_date": "2026-10-27T17:23:21.569825Z", "source_model_id": "openai/gpt-6-luna", "source_version_ids": [ 6, 8542 ], "source_task_version_id": 1109072 }, { "id": "d6a4031733610bb080d0", "model": "Gemma 4 31B", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.7583090379008747, "raw_score": 0.7583090379008747, "raw_scale": 1, "source_record": "138", "source_date": "2026-10-24T01:41:46.904Z", "source_model_id": "google/gemma-4-31b", "source_version_ids": [ 6, 8542 ], "source_task_version_id": 1109072 }, { "id": "8cd2510271575d8430c0", "model": "Qwen 3.8 Max", "organization": "QwenLM", "configuration": "Published Kaggle FACTS protocol", "score": 0.7379008746355685, "raw_score": 0.7379008746355685, "raw_scale": 1, "source_record": "177", "source_date": "2026-10-27T18:23:16.328813Z", "source_model_id": "qwen/qwen3.8-max", "source_version_ids": [ 8542 ], "source_task_version_id": 1109072 }, { "id": "38b2d03f3256502b1e9d", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.7335276967930029, "raw_score": 0.7335276967930029, "raw_scale": 1, "source_record": "187", "source_date": "2026-10-24T01:30:38.909067Z", "source_model_id": "google/gemini-4-argon", "source_version_ids": [ 8542 ], "source_task_version_id": 1109072 }, { "id": "01d54579da446ae1e75c", "model": "Grok 4.7", "organization": "xAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.6860058309037901, "raw_score": 0.6860058309037901, "raw_scale": 1, "source_record": "178", "source_date": "2026-10-24T04:45:10.992415Z", "source_model_id": "xai/grok-4.7", "source_version_ids": [ 8542 ], "source_task_version_id": 1109072 }, { "id": "80c3cd40fa35f9088b87", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.6816326530612244, "raw_score": 0.6816326530612244, "raw_scale": 1, "source_record": "168", "source_date": "2026-10-23T19:24:16.424699Z", "source_model_id": "google/gemini-3.8-flash", "source_version_ids": [ 6, 8542 ], "source_task_version_id": 1109072 }, { "id": "3068430da9e4b7a67418", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.6244897959183674, "raw_score": 0.6244897959183674, "raw_scale": 1, "source_record": "179", "source_date": "2026-10-24T02:13:08.028814Z", "source_model_id": "anthropic/claude-opus-5-5@default", "source_version_ids": [ 6, 8542 ], "source_task_version_id": 1109072 }, { "id": "210e3b160c3558185094", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.6052478134110787, "raw_score": 0.6052478134110787, "raw_scale": 1, "source_record": "155", "source_date": "2026-10-24T01:10:14.578773Z", "source_model_id": "openai/gpt-5.6-terra", "source_version_ids": [ 6, 8542 ], "source_task_version_id": 1109072 }, { "id": "d6061bbee6cf13bd7376", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.5696793002915452, "raw_score": 0.5696793002915452, "raw_scale": 1, "source_record": "188", "source_date": "2026-10-24T01:22:27.784926Z", "source_model_id": "anthropic/claude-haiku-5-5", "source_version_ids": [ 8542 ], "source_task_version_id": 1109072 }, { "id": "ad57366865126e55649e", "model": "Gemini 3.1 Pro Preview", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.5661807580174927, "raw_score": 0.5661807580174927, "raw_scale": 1, "source_record": "100", "source_date": "2026-10-24T01:34:02.275455Z", "source_model_id": "google/gemini-3.1-pro-preview", "source_version_ids": [ 6, 8542 ], "source_task_version_id": 1109072 }, { "id": "dac53c17c250fd4d4d81", "model": "GLM-5.3-Flash", "organization": "Z.ai", "configuration": "Published Kaggle FACTS protocol", "score": 0.560932944606414, "raw_score": 0.560932944606414, "raw_scale": 1, "source_record": "175", "source_date": "2026-10-27T18:22:17.390164Z", "source_model_id": "zai/glm-5.3-flash", "source_version_ids": [ 8542 ], "source_task_version_id": 1109072 }, { "id": "108c995b953c8a355611", "model": "gpt-oss-120b", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.20145772594752187, "raw_score": 0.20145772594752187, "raw_scale": 1, "source_record": "65", "source_date": "2026-10-24T03:14:13.123574Z", "source_model_id": "openai/gpt-oss-120b", "source_version_ids": [ 8542 ], "source_task_version_id": 1109072 } ] }, { "id": "google-facts-multimodal-v1", "name": "FACTS Multimodal (v1)", "group": "知识与推理", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Official Kaggle board, benchmark version 1; current mapped task version 40126. Published fractional accuracy; missing model results are omitted.", "zh": "官方 Kaggle 榜单，评测版本 1，当前映射的任务版本 40126。官方正确率；未公布成绩的模型不填分。" }, "source": { "name": "Google DeepMind / Kaggle", "benchmark_creator": "Google DeepMind", "url": "https://www.kaggle.com/benchmarks/google/facts-multimodal/leaderboard", "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmarkLeaderboard", "sha256": "d4ffe5aff17b394969aefd9bf394468ed2d032a2d014d50e1a397d447e9f1c0f", "retrieved_at": "2026-11-02T05:58:08.945813+00:00", "selector": "GetBenchmarkLeaderboard versionIdSelector.id=6 or 157; rows.results[taskVersionId=40126].numericResult.value", "kind": "official_leaderboard", "status": "current_snapshot", "request_body": { "versionIdentifier": { "versionIdSelector": { "id": 6 } } }, "artifacts": [ { "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmarkLeaderboard", "sha256": "018282d50d3a1e382a14e91a6f962c3215655811a0a5aad94b630821e72c708e", "retrieved_at": "2026-11-02T05:58:34.782504+00:00", "request_body": { "versionIdentifier": { "versionIdSelector": { "id": 157 } } } } ] }, "results": [ { "id": "80c3cd40fa35f9088b87", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.5069953364423717, "raw_score": 0.5069953364423717, "raw_scale": 1, "source_record": "168", "source_date": "2026-09-04T01:26:51.367176Z", "source_model_id": "google/gemini-3.8-flash", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "3f9807cb9ae9fb6c3094", "model": "Gemini 3.7 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.49833666001330673, "raw_score": 0.49833666001330673, "raw_scale": 1, "source_record": "164", "source_date": "2026-08-15T01:47:54.831882Z", "source_model_id": "google/gemini-3.7-flash", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "2811745d7b8d8874f6e6", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.48462566844919786, "raw_score": 0.48462566844919786, "raw_scale": 1, "source_record": "186", "source_date": "2026-10-05T22:48:00.516991Z", "source_model_id": "openai/gpt-6.1-sol", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "3068430da9e4b7a67418", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.48367346938775513, "raw_score": 0.48367346938775513, "raw_scale": 1, "source_record": "179", "source_date": "2026-09-25T01:05:59.008093Z", "source_model_id": "anthropic/claude-opus-5-5@default", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "734d0759cdb4e0d0a35e", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.4773030707610147, "raw_score": 0.4773030707610147, "raw_scale": 1, "source_record": "170", "source_date": "2026-09-12T00:43:16.283649Z", "source_model_id": "openai/gpt-6-astra", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "ff2ccb6ba423d356bd54", "model": "Gemini 3.6 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.4720744680851064, "raw_score": 0.4720744680851064, "raw_scale": 1, "source_record": "159", "source_date": "2026-07-23T08:03:41.060957Z", "source_model_id": "google/gemini-3.6-flash", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "031b4af5197ec30a926f", "model": "Gemini 2.5 Pro", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.46860820194999997, "raw_score": 0.46860820194999997, "raw_scale": 1, "source_record": "51", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "google/gemini-2.5-pro", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "d4ee9f58e5860574ca98", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.4576948700866089, "raw_score": 0.4576948700866089, "raw_scale": 1, "source_record": "142", "source_date": "", "source_model_id": "openai/gpt-5.5-2026-04-23", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "a68b412c4282555f1554", "model": "GPT-5", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.44078298395000004, "raw_score": 0.44078298395000004, "raw_scale": 1, "source_record": "64", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "openai/gpt-5-2025-08-07", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "7b69759630f869f27238", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.43057409879839786, "raw_score": 0.43057409879839786, "raw_scale": 1, "source_record": "180", "source_date": "2026-09-25T19:55:58.853109Z", "source_model_id": "openai/gpt-6-sol", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "a46e37632fa6ca51a13f", "model": "GPT-5.1", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.418010359, "raw_score": 0.418010359, "raw_scale": 1, "source_record": "82", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "openai/gpt-5.1-2025-11-13", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "41cfc0d1f2d127b04555", "model": "Gemini 2.5 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.4160572294460662, "raw_score": 0.4160572294460662, "raw_scale": 1, "source_record": "52", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "google/gemini-2.5-flash", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "349c41201b62db851192", "model": "GPT-5 mini", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.4147483514, "raw_score": 0.4147483514, "raw_scale": 1, "source_record": "78", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "openai/gpt-5-mini-2025-08-07", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "0a5b046d07f6f971b777", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.4143904063957361, "raw_score": 0.4143904063957361, "raw_scale": 1, "source_record": "146", "source_date": "", "source_model_id": "google/gemini-3.5-flash", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "ad57366865126e55649e", "model": "Gemini 3.1 Pro Preview", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.4134228187919463, "raw_score": 0.4134228187919463, "raw_scale": 1, "source_record": "100", "source_date": "", "source_model_id": "google/gemini-3.1-pro-preview", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "bdd2d3af3a5a1213497d", "model": "Gemini 3 Flash Preview", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.4128256513026052, "raw_score": 0.4128256513026052, "raw_scale": 1, "source_record": "87", "source_date": "", "source_model_id": "google/gemini-3-flash-preview", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "d2f483672c0239f6d7dd", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.40975935828877, "raw_score": 0.40975935828877, "raw_scale": 1, "source_record": "133", "source_date": "", "source_model_id": "openai/gpt-5.4-2026-03-05", "source_version_ids": [ 157 ], "source_task_version_id": 40126 }, { "id": "c6f3ac57944a531490cd", "model": "GPT-4.1", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.4054397825191458, "raw_score": 0.4054397825191458, "raw_scale": 1, "source_record": "33", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "openai/gpt-4.1-2025-04-14", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "e29c9c180c6279b0b02a", "model": "o3", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.40204709409064726, "raw_score": 0.40204709409064726, "raw_scale": 1, "source_record": "32", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "openai/o3-2025-04-16", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "1d28c120568c10e19b9d", "model": "Claude Opus 4.8", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.397719651240778, "raw_score": 0.397719651240778, "raw_scale": 1, "source_record": "147", "source_date": "", "source_model_id": "anthropic/claude-opus-4-8@default", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "b4944c6ff08dc6f43da2", "model": "Claude Opus 4.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.39721688358286694, "raw_score": 0.39721688358286694, "raw_scale": 1, "source_record": "85", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "anthropic/claude-opus-4-5@20251101", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "e3d6c4d4599e00882384", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.3966442953020134, "raw_score": 0.3966442953020134, "raw_scale": 1, "source_record": "94", "source_date": "", "source_model_id": "openai/gpt-5.2-2025-12-11", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "13671077b66a29874a25", "model": "Gemini 3.1 Flash-Lite Preview", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.3935957304869913, "raw_score": 0.3935957304869913, "raw_scale": 1, "source_record": "135", "source_date": "", "source_model_id": "google/gemini-3.1-flash-lite-preview", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "29db0c6782dbd5000559", "model": "Claude Opus 4.6", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.3675385647216633, "raw_score": 0.3675385647216633, "raw_scale": 1, "source_record": "98", "source_date": "", "source_model_id": "anthropic/claude-opus-4-6@default", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "dbae772db29058a88f9b", "model": "Claude Opus 4.7", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.3584277148567621, "raw_score": 0.3584277148567621, "raw_scale": 1, "source_record": "140", "source_date": "", "source_model_id": "anthropic/claude-opus-4-7@default", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "a512db2741cd20693e4b", "model": "Gemini 3.5 Flash-Lite", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.3583776595744681, "raw_score": 0.3583776595744681, "raw_scale": 1, "source_record": "160", "source_date": "2026-07-23T01:51:26.686178Z", "source_model_id": "google/gemini-3.5-flash-lite", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "620c9c332101a5bae955", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.35809806835066865, "raw_score": 0.35809806835066865, "raw_scale": 1, "source_record": "153", "source_date": "2026-08-12T20:28:57.691076Z", "source_model_id": "openai/gpt-5.6-luna", "source_version_ids": [ 6 ], "source_task_version_id": 40126 }, { "id": "eb1e33e8a81b697b7585", "model": "o4 mini", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.35432962139987784, "raw_score": 0.35432962139987784, "raw_scale": 1, "source_record": "31", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "openai/o4-mini-2025-04-16", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "580811fa95269f3ecd4f", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.342456608811749, "raw_score": 0.342456608811749, "raw_scale": 1, "source_record": "181", "source_date": "2026-09-25T20:06:55.338088Z", "source_model_id": "openai/gpt-6-luna", "source_version_ids": [ 6 ], "source_task_version_id": 40126 }, { "id": "a21855da08cb102d1d21", "model": "Claude Opus 4.1", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.3331880154823186, "raw_score": 0.3331880154823186, "raw_scale": 1, "source_record": "68", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "anthropic/claude-opus-4-1@20250805", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "8c1f1046219ddd216a02", "model": "Claude Sonnet 4.6", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.3255033557046979, "raw_score": 0.3255033557046979, "raw_scale": 1, "source_record": "99", "source_date": "", "source_model_id": "anthropic/claude-sonnet-4-6@default", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "bbb965ab0c80d6538cf2", "model": "Claude Sonnet 4.5 (thinking)", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.3106767038, "raw_score": 0.3106767038, "raw_scale": 1, "source_record": "83", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "71ee45a3c0db9a9865f7", "model": "Claude Sonnet 4", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.28625754285, "raw_score": 0.28625754285, "raw_scale": 1, "source_record": "44", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "anthropic/claude-sonnet-4@20250514", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "c837649cce43f2729138", "model": "Grok 4", "organization": "xAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.2570295860890035, "raw_score": 0.2570295860890035, "raw_scale": 1, "source_record": "57", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "xai/grok-4-0709", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "d6a4031733610bb080d0", "model": "Gemma 4 31B", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.2471685542971352, "raw_score": 0.2471685542971352, "raw_scale": 1, "source_record": "138", "source_date": "", "source_model_id": "google/gemma-4-31b", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "59e19706d51d39f66711", "model": "Gemma 3 27B", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.2463069933, "raw_score": 0.2463069933, "raw_scale": 1, "source_record": "28", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "google/gemma-3-27b", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "8d27ba37c5d810106b55", "model": "Gemma 4 26B A4B", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.2385076615589607, "raw_score": 0.2385076615589607, "raw_scale": 1, "source_record": "139", "source_date": "", "source_model_id": "google/gemma-4-26b-a4b", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "86e50149658661312a9e", "model": "Mistral Small 3.1", "organization": "Mistral AI", "configuration": "Published Kaggle FACTS protocol", "score": 0.21170987614999998, "raw_score": 0.21170987614999998, "raw_scale": 1, "source_record": "34", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "mistralai/mistral-small-2503", "source_version_ids": [ 157 ], "source_task_version_id": 40126 }, { "id": "35135aaa6cc23891b40c", "model": "Gemma 3 12B", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.2080870178, "raw_score": 0.2080870178, "raw_scale": 1, "source_record": "29", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "google/gemma-3-12b", "source_version_ids": [ 157 ], "source_task_version_id": 40126 }, { "id": "f74efabef12ea619e30b", "model": "Grok 4 Fast Reasoning", "organization": "xAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.1773702850016814, "raw_score": 0.1773702850016814, "raw_scale": 1, "source_record": "76", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "xai/grok-4-fast-reasoning", "source_version_ids": [ 6, 157 ], "source_task_version_id": 40126 }, { "id": "670671cd97404156226e", "model": "Gemma 3 4B", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.1363804586, "raw_score": 0.1363804586, "raw_scale": 1, "source_record": "27", "source_date": "2025-12-07T00:00:00Z", "source_model_id": "google/gemma-3-4b", "source_version_ids": [ 157 ], "source_task_version_id": 40126 } ] }, { "id": "google-facts-parametric-v2", "name": "FACTS Parametric (v2)", "group": "知识与推理", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Official Kaggle board, benchmark version 2; current mapped task version 1118505. Published fractional accuracy; missing model results are omitted.", "zh": "官方 Kaggle 榜单，评测版本 2，当前映射的任务版本 1118505。官方正确率；未公布成绩的模型不填分。" }, "source": { "name": "Google DeepMind / Kaggle", "benchmark_creator": "Google DeepMind", "url": "https://www.kaggle.com/benchmarks/google/facts-parametric/leaderboard", "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmarkLeaderboard", "sha256": "d4ffe5aff17b394969aefd9bf394468ed2d032a2d014d50e1a397d447e9f1c0f", "retrieved_at": "2026-11-02T05:58:08.945813+00:00", "selector": "GetBenchmarkLeaderboard versionIdSelector.id=6 or 8656; rows.results[taskVersionId=1118505].numericResult.value", "kind": "official_leaderboard", "status": "current_snapshot", "request_body": { "versionIdentifier": { "versionIdSelector": { "id": 6 } } }, "artifacts": [ { "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmarkLeaderboard", "sha256": "5a94e2294e8d11a9d56c3c476bdd6e5ca50828cc087a3685fe61c05ea35184c7", "retrieved_at": "2026-11-02T05:58:34.907228+00:00", "request_body": { "versionIdentifier": { "versionIdSelector": { "id": 8656 } } } } ] }, "results": [ { "id": "734d0759cdb4e0d0a35e", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8382446134347276, "raw_score": 0.8382446134347276, "raw_scale": 1, "source_record": "170", "source_date": "2026-10-26T10:36:05.346607Z", "source_model_id": "openai/gpt-6-astra", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "2811745d7b8d8874f6e6", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8376108998732572, "raw_score": 0.8376108998732572, "raw_scale": 1, "source_record": "186", "source_date": "2026-10-26T10:37:32.627328Z", "source_model_id": "openai/gpt-6.1-sol", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "1d0ebea552eb43d0b1e1", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8109949302915082, "raw_score": 0.8109949302915082, "raw_scale": 1, "source_record": "154", "source_date": "2026-10-26T10:35:17.381362Z", "source_model_id": "openai/gpt-5.6-sol", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "80c3cd40fa35f9088b87", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.7870722433460076, "raw_score": 0.7870722433460076, "raw_scale": 1, "source_record": "168", "source_date": "2026-10-25T18:20:56.269Z", "source_model_id": "google/gemini-3.8-flash", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "7b69759630f869f27238", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.7612484157160964, "raw_score": 0.7612484157160964, "raw_scale": 1, "source_record": "180", "source_date": "2026-10-26T10:28:26.640299Z", "source_model_id": "openai/gpt-6-sol", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "ff2ccb6ba423d356bd54", "model": "Gemini 3.6 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.7389100126742713, "raw_score": 0.7389100126742713, "raw_scale": 1, "source_record": "159", "source_date": "2026-10-26T10:31:18.121064Z", "source_model_id": "google/gemini-3.6-flash", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "bdd2d3af3a5a1213497d", "model": "Gemini 3 Flash Preview", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.7240177439797212, "raw_score": 0.7240177439797212, "raw_scale": 1, "source_record": "87", "source_date": "2026-10-26T10:25:32.044462Z", "source_model_id": "google/gemini-3-flash-preview", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "38b2d03f3256502b1e9d", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.7224334600760456, "raw_score": 0.7224334600760456, "raw_scale": 1, "source_record": "187", "source_date": "2026-10-26T10:37:26.091398Z", "source_model_id": "google/gemini-4-argon", "source_version_ids": [ 8656 ], "source_task_version_id": 1118505 }, { "id": "3068430da9e4b7a67418", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.5872940430925222, "raw_score": 0.5872940430925222, "raw_scale": 1, "source_record": "179", "source_date": "2026-10-26T10:37:41.492282Z", "source_model_id": "anthropic/claude-opus-5-5@default", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "210e3b160c3558185094", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.5758871989860583, "raw_score": 0.5758871989860583, "raw_scale": 1, "source_record": "155", "source_date": "2026-10-26T10:29:32.115363Z", "source_model_id": "openai/gpt-5.6-terra", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "580811fa95269f3ecd4f", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.5698669201520913, "raw_score": 0.5698669201520913, "raw_scale": 1, "source_record": "181", "source_date": "2026-10-26T10:40:15.210283Z", "source_model_id": "openai/gpt-6-luna", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "01d54579da446ae1e75c", "model": "Grok 4.7", "organization": "xAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.5427756653992395, "raw_score": 0.5427756653992395, "raw_scale": 1, "source_record": "178", "source_date": "2026-10-26T12:31:59.844763Z", "source_model_id": "xai/grok-4.7", "source_version_ids": [ 8656 ], "source_task_version_id": 1118505 }, { "id": "620c9c332101a5bae955", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.5063371356147022, "raw_score": 0.5063371356147022, "raw_scale": 1, "source_record": "153", "source_date": "2026-10-26T10:32:25.294813Z", "source_model_id": "openai/gpt-5.6-luna", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "bb668ca95563216088b9", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.48748415716096327, "raw_score": 0.48748415716096327, "raw_scale": 1, "source_record": "161", "source_date": "2026-10-26T10:51:59.643999Z", "source_model_id": "anthropic/claude-opus-5@default", "source_version_ids": [ 8656 ], "source_task_version_id": 1118505 }, { "id": "bc52dd634277c4a34a2d", "model": "Grok 4.6", "organization": "xAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.44470849176172367, "raw_score": 0.44470849176172367, "raw_scale": 1, "source_record": "165", "source_date": "2026-10-26T13:07:09.633546Z", "source_model_id": "xai/grok-4.6", "source_version_ids": [ 8656 ], "source_task_version_id": 1118505 }, { "id": "a512db2741cd20693e4b", "model": "Gemini 3.5 Flash-Lite", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.43029150823827633, "raw_score": 0.43029150823827633, "raw_scale": 1, "source_record": "160", "source_date": "2026-10-26T10:20:13.787485Z", "source_model_id": "google/gemini-3.5-flash-lite", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "0fecf9247f3ddc84db8a", "model": "Grok 4.5", "organization": "xAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.4245880861850444, "raw_score": 0.4245880861850444, "raw_scale": 1, "source_record": "156", "source_date": "2026-10-26T10:45:11.633546Z", "source_model_id": "xai/grok-4.5-0708", "source_version_ids": [ 6, 8656 ], "source_task_version_id": 1118505 }, { "id": "61a229bae1e90331edd9", "model": "Claude Sonnet 5.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.35107731305449935, "raw_score": 0.35107731305449935, "raw_scale": 1, "source_record": "185", "source_date": "2026-10-26T10:39:37.881215Z", "source_model_id": "anthropic/claude-sonnet-5-5@default", "source_version_ids": [ 8656 ], "source_task_version_id": 1118505 }, { "id": "043066daf2109523a749", "model": "Claude Sonnet 5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.2046894803548796, "raw_score": 0.2046894803548796, "raw_scale": 1, "source_record": "152", "source_date": "2026-10-26T10:31:08.758655Z", "source_model_id": "anthropic/claude-sonnet-5@default", "source_version_ids": [ 8656 ], "source_task_version_id": 1118505 }, { "id": "d6061bbee6cf13bd7376", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.1308618504435995, "raw_score": 0.1308618504435995, "raw_scale": 1, "source_record": "188", "source_date": "2026-10-26T10:33:27.593476Z", "source_model_id": "anthropic/claude-haiku-5-5", "source_version_ids": [ 8656 ], "source_task_version_id": 1118505 } ] }, { "id": "google-facts-search-v2", "name": "FACTS Search V2", "group": "知识与推理", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Official Kaggle board, benchmark version 2; current mapped task version 41923. Published fractional accuracy; missing model results are omitted.", "zh": "官方 Kaggle 榜单，评测版本 2，当前映射的任务版本 41923。官方正确率；未公布成绩的模型不填分。" }, "source": { "name": "Google DeepMind / Kaggle", "benchmark_creator": "Google DeepMind", "url": "https://www.kaggle.com/benchmarks/google/facts-search/leaderboard", "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmarkLeaderboard", "sha256": "d4ffe5aff17b394969aefd9bf394468ed2d032a2d014d50e1a397d447e9f1c0f", "retrieved_at": "2026-11-02T05:58:08.945813+00:00", "selector": "GetBenchmarkLeaderboard versionIdSelector.id=6 or 6295; rows.results[taskVersionId=41923].numericResult.value", "kind": "official_leaderboard", "status": "current_snapshot", "request_body": { "versionIdentifier": { "versionIdSelector": { "id": 6 } } }, "artifacts": [ { "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmarkLeaderboard", "sha256": "698d3fd664b65107b6ec034e5164433e86dc7ba57724e0ae7667e052d66a2cae", "retrieved_at": "2026-11-02T05:58:34.762899+00:00", "request_body": { "versionIdentifier": { "versionIdSelector": { "id": 6295 } } } } ] }, "results": [ { "id": "734d0759cdb4e0d0a35e", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8991507430997877, "raw_score": 0.8991507430997877, "raw_scale": 1, "source_record": "170", "source_date": "2026-09-15T23:13:18.205059Z", "source_model_id": "openai/gpt-6-astra", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "1d0ebea552eb43d0b1e1", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8848195329087049, "raw_score": 0.8848195329087049, "raw_scale": 1, "source_record": "154", "source_date": "2026-07-24T01:44:33.938998Z", "source_model_id": "openai/gpt-5.6-sol", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "2811745d7b8d8874f6e6", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8816348195329087, "raw_score": 0.8816348195329087, "raw_scale": 1, "source_record": "186", "source_date": "2026-10-05T19:42:35.672762Z", "source_model_id": "openai/gpt-6.1-sol", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "3068430da9e4b7a67418", "model": "Claude Opus 5.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.8651804670912951, "raw_score": 0.8651804670912951, "raw_scale": 1, "source_record": "179", "source_date": "2026-09-24T20:20:26.693106Z", "source_model_id": "anthropic/claude-opus-5-5@default", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "7b69759630f869f27238", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8614649681528662, "raw_score": 0.8614649681528662, "raw_scale": 1, "source_record": "180", "source_date": "2026-09-29T01:12:37.699767Z", "source_model_id": "openai/gpt-6-sol", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "bb668ca95563216088b9", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.8527379053694844, "raw_score": 0.8527379053694844, "raw_scale": 1, "source_record": "161", "source_date": "2026-07-27T21:19:33.380608Z", "source_model_id": "anthropic/claude-opus-5@default", "source_version_ids": [ 6295 ], "source_task_version_id": 41923 }, { "id": "38b2d03f3256502b1e9d", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.8401924104756815, "raw_score": 0.8401924104756815, "raw_scale": 1, "source_record": "187", "source_date": "2026-10-09T04:30:35.929494Z", "source_model_id": "google/gemini-4-argon", "source_version_ids": [ 6295 ], "source_task_version_id": 41923 }, { "id": "ad57366865126e55649e", "model": "Gemini 3.1 Pro Preview", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.8370488322717622, "raw_score": 0.8370488322717622, "raw_scale": 1, "source_record": "100", "source_date": "2026-07-25T00:57:28.232365Z", "source_model_id": "google/gemini-3.1-pro-preview", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "210e3b160c3558185094", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8174097664543525, "raw_score": 0.8174097664543525, "raw_scale": 1, "source_record": "155", "source_date": "2026-07-23T17:37:13.155555Z", "source_model_id": "openai/gpt-5.6-terra", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "580811fa95269f3ecd4f", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.8142250530785563, "raw_score": 0.8142250530785563, "raw_scale": 1, "source_record": "181", "source_date": "2026-09-29T01:09:59.270469Z", "source_model_id": "openai/gpt-6-luna", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "bdd2d3af3a5a1213497d", "model": "Gemini 3 Flash Preview", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.8092948717948718, "raw_score": 0.8092948717948718, "raw_scale": 1, "source_record": "87", "source_date": "2026-07-18T00:14:09.964744Z", "source_model_id": "google/gemini-3-flash-preview", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "3f9807cb9ae9fb6c3094", "model": "Gemini 3.7 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.8089171974522293, "raw_score": 0.8089171974522293, "raw_scale": 1, "source_record": "164", "source_date": "2026-08-18T23:19:52.163758Z", "source_model_id": "google/gemini-3.7-flash", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "1d28c120568c10e19b9d", "model": "Claude Opus 4.8", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.79596174282678, "raw_score": 0.79596174282678, "raw_scale": 1, "source_record": "147", "source_date": "2026-07-24T15:13:00.672942Z", "source_model_id": "anthropic/claude-opus-4-8@default", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "0fecf9247f3ddc84db8a", "model": "Grok 4.5", "organization": "xAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.7943676939426142, "raw_score": 0.7943676939426142, "raw_scale": 1, "source_record": "156", "source_date": "2026-07-23T17:56:45.063016Z", "source_model_id": "xai/grok-4.5-0708", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "620c9c332101a5bae955", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Published Kaggle FACTS protocol", "score": 0.7935244161358811, "raw_score": 0.7935244161358811, "raw_scale": 1, "source_record": "153", "source_date": "2026-07-24T01:28:13.386206Z", "source_model_id": "openai/gpt-5.6-luna", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "d6061bbee6cf13bd7376", "model": "Claude Haiku 5.5", "organization": "Anthropic", "configuration": "Published Kaggle FACTS protocol", "score": 0.7749469214437368, "raw_score": 0.7749469214437368, "raw_scale": 1, "source_record": "188", "source_date": "2026-10-09T17:45:02.119077Z", "source_model_id": "anthropic/claude-haiku-5-5", "source_version_ids": [ 6295 ], "source_task_version_id": 41923 }, { "id": "80c3cd40fa35f9088b87", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.7632696390658175, "raw_score": 0.7632696390658175, "raw_scale": 1, "source_record": "168", "source_date": "2026-09-03T21:05:03.812565Z", "source_model_id": "google/gemini-3.8-flash", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "ff2ccb6ba423d356bd54", "model": "Gemini 3.6 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.6819148936170213, "raw_score": 0.6819148936170213, "raw_scale": 1, "source_record": "159", "source_date": "2026-07-23T17:55:45.247336Z", "source_model_id": "google/gemini-3.6-flash", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "d6a4031733610bb080d0", "model": "Gemma 4 31B", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.6814044213263979, "raw_score": 0.6814044213263979, "raw_scale": 1, "source_record": "138", "source_date": "2026-07-27T18:22:01.942202Z", "source_model_id": "google/gemma-4-31b", "source_version_ids": [ 6 ], "source_task_version_id": 41923 }, { "id": "0a5b046d07f6f971b777", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.6427795874049945, "raw_score": 0.6427795874049945, "raw_scale": 1, "source_record": "146", "source_date": "2026-07-23T03:03:35.902251Z", "source_model_id": "google/gemini-3.5-flash", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "a512db2741cd20693e4b", "model": "Gemini 3.5 Flash-Lite", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.6342887473460722, "raw_score": 0.6342887473460722, "raw_scale": 1, "source_record": "160", "source_date": "2026-07-23T14:27:04.613203Z", "source_model_id": "google/gemini-3.5-flash-lite", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "8d27ba37c5d810106b55", "model": "Gemma 4 26B A4B", "organization": "Google", "configuration": "Published Kaggle FACTS protocol", "score": 0.6031317494600432, "raw_score": 0.6031317494600432, "raw_scale": 1, "source_record": "139", "source_date": "2026-07-28T00:20:01.595322Z", "source_model_id": "google/gemma-4-26b-a4b", "source_version_ids": [ 6, 6295 ], "source_task_version_id": 41923 }, { "id": "49d180ecf56132819571", "model": "Qwen 3 235B A22B Instruct 2506", "organization": "QwenLM", "configuration": "Published Kaggle FACTS protocol", "score": 0.5335106382978724, "raw_score": 0.5335106382978724, "raw_scale": 1, "source_record": "67", "source_date": "2026-07-27T16:37:59.587809Z", "source_model_id": "qwen/qwen3-235b-a22b-instruct-2507", "source_version_ids": [ 6295 ], "source_task_version_id": 41923 } ] }, { "id": "tau3-banking-sierra", "name": "τ³-Banking (Sierra)", "group": "Agent 与工具", "unit": "%", "metric": "Pass@1", "protocol": { "en": "Sierra standard submissions, banking_knowledge, v1.0.1 re-grading. Pass@1; retrieval, reasoning and user-simulator settings remain per result.", "zh": "Sierra 标准提交，banking_knowledge、v1.0.1 重评分；使用 pass@1，逐条保留检索方式、推理与用户模拟器版本。" }, "source": { "name": "Sierra", "benchmark_creator": "Sierra", "url": "https://taubench.com/leaderboard?benchmark=knowledge", "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/manifest.json", "sha256": "9baaaf615efc04442fc385c651cb3fa0b87d5cb930eda001670d63e173e37b48", "retrieved_at": "2026-11-02T03:36:07.400733+00:00", "selector": "manifest.submissions -> submission.json; Sierra/standard/v1.0.1; results.banking_knowledge.pass_1", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/claude-opus-4-5_sierra_2026-02-26/submission.json", "sha256": "9c3b8b400e21d370cea27caec3643bfe6fd8385be9ed5135b585d0b976b6e34f", "retrieved_at": "2026-11-02T03:40:35.449784+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/claude-sonnet-4-5_sierra_2026-02-26/submission.json", "sha256": "e5ce9f12ab9e616db52e973e90b48144b7bafd29e35c4fd6c3e7fde573a9b8d8", "retrieved_at": "2026-11-02T03:40:35.261536+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/gpt-5-2_sierra_2026-02-26/submission.json", "sha256": "343687b822edbcbff42a4d4632ecc6288a58d9817a7a02c8533081c5709c0ac6", "retrieved_at": "2026-11-02T03:40:40.681992+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/gpt-5-2-none_sierra_2026-02-26/submission.json", "sha256": "e1be4bac436e8115f9253697d68b71867f17cfcb2bef5dfa6a756c0a89a70dab", "retrieved_at": "2026-11-02T03:40:40.651893+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/glm-5-think_sierra_2026-03-02/submission.json", "sha256": "fa27b9a1a7a4c8b15517ec1329ef21fcc700be2ca12f21659b60aeda99c7de8f", "retrieved_at": "2026-11-02T03:40:40.658472+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/qwen3.5-397b-a17b-think_sierra_2026-03-02/submission.json", "sha256": "2cfb2c7d2a46e9f22154fa153dcdf155d50dbd5bf86233ecf564eccca1e5913e", "retrieved_at": "2026-11-02T03:40:40.658511+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/gemini-3-flash_sierra_2026-03-02/submission.json", "sha256": "d1b7d1fc5d789662fb6eeb4a5ec01948295db397d2313df42c43c7fd8caf3637", "retrieved_at": "2026-11-02T03:40:40.672937+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/gemini-3-pro_sierra_2026-03-02/submission.json", "sha256": "f5c7b2d432d4efc6d6c1533f87e61ae1673336579df9f5f5cf986c277f0afd18", "retrieved_at": "2026-11-02T03:40:40.637189+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/gpt-5-4_sierra_2026-03-25/submission.json", "sha256": "47fa11691a745682b86ffc91e1a57aef6d6d7cebe60df66820ebb310b518b0ab", "retrieved_at": "2026-11-02T03:40:42.333441+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/gpt-5-5_sierra_2026-05-05/submission.json", "sha256": "24247d74b2a942da4c8f85503d0f8b95ba6e2d5df7158c2adf15049056d8b7f4", "retrieved_at": "2026-11-02T03:40:42.313237+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/claude-opus-4-6_sierra_2026-05-05/submission.json", "sha256": "880ea58f76664a0f0e06c3117c6a8b75c4007149d06cc3de04f852f62487f3f3", "retrieved_at": "2026-11-02T03:40:42.604087+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/claude-opus-4-7_sierra_2026-05-05/submission.json", "sha256": "7ec10475f58a7d24c55548df9b8a04b6e6105dc25a6927f618c487185f28eca9", "retrieved_at": "2026-11-02T03:40:42.470381+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/gemini-2-5-pro_sierra_2026-05-05/submission.json", "sha256": "b14f65b54ac7b733550cb1e04b986d17b7ff877de6ff79e94e8e98998035de6f", "retrieved_at": "2026-11-02T03:40:48.057496+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/gemini-3-1-pro-preview_sierra_2026-05-05/submission.json", "sha256": "16ada360be015c129cc704ab283535144305df1b1cb911750bb2ec6a2fde53ac", "retrieved_at": "2026-11-02T03:40:48.237382+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/grok-4-fast_sierra_2026-05-05/submission.json", "sha256": "e310d5fa844297cbb9b3ff0755f9784ddeae8daf99624f1003a8d0bdd973c237", "retrieved_at": "2026-11-02T03:40:48.156318+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/grok-4-1-fast_sierra_2026-05-05/submission.json", "sha256": "24efda601208398e37890ee127e37190b2080dc8eb34aa52fac0ee7f32ae1e08", "retrieved_at": "2026-11-02T03:40:48.076204+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/grok-4-2_sierra_2026-05-05/submission.json", "sha256": "6187e4b645a9de92f40ba0d6b800c13200ef39c5bc95062d5ae8122251459fc6", "retrieved_at": "2026-11-02T03:40:48.117142+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/gpt-5-6-sol_sierra_2026-08-04/submission.json", "sha256": "0394128dc04d58ff77846e404ce588d786988a5c2ffcb69a8af09f19c46c97a4", "retrieved_at": "2026-11-02T03:40:48.076156+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/claude-opus-4-8_sierra_2026-08-04/submission.json", "sha256": "13052291f58a74e601c4e577bce110a7fad93a126edaa87fb483e49df0823054", "retrieved_at": "2026-11-02T03:40:49.585965+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/claude-fable-5_sierra_2026-08-04/submission.json", "sha256": "f5cb61669a7f9665a006031ae1779b59230dcb4527cf763cfc3d2c771878166b", "retrieved_at": "2026-11-02T03:40:49.565589+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/muse-spark-1-1_sierra_2026-08-04/submission.json", "sha256": "5ec4d2b119542bb40358374e3408515dccf5f0b3dace60de7936a20cb9bf13da", "retrieved_at": "2026-11-02T03:40:49.555771+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/kimi-k3_sierra_2026-08-04/submission.json", "sha256": "71e1ac4413f59ceac04851843319f73b485280a756f3b69fb5f47836d65a52b6", "retrieved_at": "2026-11-02T03:40:49.546881+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/glm-5-2_sierra_2026-08-04/submission.json", "sha256": "58b36fd68b92eac5c9e72efce3952aa4004db2b7f6f520c2bcf2908c4845e801", "retrieved_at": "2026-11-02T03:40:49.549989+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/grok-4-5_sierra_2026-08-04/submission.json", "sha256": "fb6414a4c05756d2ea00289684680f22fe3111a1ed69cf8d77c1071b1075bcb1", "retrieved_at": "2026-11-02T03:40:49.607413+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/inkling_sierra_2026-08-04/submission.json", "sha256": "f5d7b5fae9986619bc3258d1be25b3577f46d4e61a1b6c8853933710eb3169b9", "retrieved_at": "2026-11-02T03:40:55.320545+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/claude-opus-5_sierra_2026-08-04/submission.json", "sha256": "4ab17f3f2e77cae8b2b4e4ba13413f7e2168104a772f25afe9d071950ed24ba2", "retrieved_at": "2026-11-02T03:40:55.208332+00:00" }, { "data_url": "https://sierra-tau-bench-public.s3.us-west-2.amazonaws.com/submissions/qwen3-8-max_sierra_2026-08-04/submission.json", "sha256": "afd5525e447a221f6997b6af8a6a72536acda81b0e7775fd7124836a13a9ad76", "retrieved_at": "2026-11-02T03:40:55.190950+00:00" } ] }, "results": [ { "id": "330a124ec554e615c9ab", "model": "Qwen 3.8 Max", "organization": "Qwen", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=xhigh; user simulator=gpt-5.2; v1.0.1", "score": 0.5515463917525774, "raw_score": 55.154639175257735, "raw_scale": 100, "source_record": "qwen3-8-max_sierra_2026-08-04", "source_date": "2026-08-03" }, { "id": "40c4f207693fdafe99ae", "model": "Claude Opus 5", "organization": "Anthropic", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=max; user simulator=gpt-5.2; v1.0.1", "score": 0.48711340206185566, "raw_score": 48.71134020618557, "raw_scale": 100, "source_record": "claude-opus-5_sierra_2026-08-04", "source_date": "2026-08-03" }, { "id": "1b0f94ae99a11091ee03", "model": "Grok 4.5", "organization": "xAI", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.4793814432989691, "raw_score": 47.93814432989691, "raw_scale": 100, "source_record": "grok-4-5_sierra_2026-08-04", "source_date": "2026-07-24" }, { "id": "7938f5b3464968d20032", "model": "GPT-5.6-sol", "organization": "OpenAI", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=xhigh; user simulator=gpt-5.2; v1.0.1", "score": 0.46907216494845355, "raw_score": 46.90721649484536, "raw_scale": 100, "source_record": "gpt-5-6-sol_sierra_2026-08-04", "source_date": "2026-07-22" }, { "id": "61e569912f783e12372f", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=xhigh; user simulator=gpt-5.2; v1.0.1", "score": 0.44587628865979384, "raw_score": 44.58762886597938, "raw_scale": 100, "source_record": "gpt-5-5_sierra_2026-05-05", "source_date": "2026-07-22" }, { "id": "04dae481b6e1b376119e", "model": "Muse Spark 1.1", "organization": "Meta", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=xhigh; user simulator=gpt-5.2; v1.0.1", "score": 0.40463917525773196, "raw_score": 40.4639175257732, "raw_scale": 100, "source_record": "muse-spark-1-1_sierra_2026-08-04", "source_date": "2026-07-23" }, { "id": "4a16653e6768011702d4", "model": "Claude Opus 4.7", "organization": "Anthropic", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=max; user simulator=gpt-5.2; v1.0.1", "score": 0.4020618556701031, "raw_score": 40.20618556701031, "raw_scale": 100, "source_record": "claude-opus-4-7_sierra_2026-05-05", "source_date": "2026-07-23" }, { "id": "0df521afd64baf9f26d9", "model": "Claude Fable 5", "organization": "Anthropic", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=max; user simulator=gpt-5.2; v1.0.1", "score": 0.3969072164948454, "raw_score": 39.69072164948454, "raw_scale": 100, "source_record": "claude-fable-5_sierra_2026-08-04", "source_date": "2026-07-23" }, { "id": "a8dff2f922ea834c5da5", "model": "Claude Opus 4.8", "organization": "Anthropic", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=max; user simulator=gpt-5.2; v1.0.1", "score": 0.3969072164948454, "raw_score": 39.69072164948454, "raw_scale": 100, "source_record": "claude-opus-4-8_sierra_2026-08-04", "source_date": "2026-07-23" }, { "id": "684df6f404244403f277", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=xhigh; user simulator=gpt-5.2; v1.0.1", "score": 0.3943, "raw_score": 39.43, "raw_scale": 100, "source_record": "gpt-5-4_sierra_2026-03-25", "source_date": "2026-05-06" }, { "id": "7fc7508e935433ebb3f2", "model": "GLM-5.2", "organization": "Z.ai", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=xhigh; user simulator=gpt-5.2; v1.0.1", "score": 0.3711340206185567, "raw_score": 37.11340206185567, "raw_scale": 100, "source_record": "glm-5-2_sierra_2026-08-04", "source_date": "2026-07-24" }, { "id": "88572e7c5229f266f695", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=max; user simulator=gpt-5.2; v1.0.1", "score": 0.3711340206185567, "raw_score": 37.11340206185567, "raw_scale": 100, "source_record": "kimi-k3_sierra_2026-08-04", "source_date": "2026-07-24" }, { "id": "10e2a028894949019eb0", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.3222, "raw_score": 32.22, "raw_scale": 100, "source_record": "gpt-5-2_sierra_2026-02-26", "source_date": "2026-05-05" }, { "id": "2690b9734a9a54139aa0", "model": "Claude Opus 4.6", "organization": "Anthropic", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=max; user simulator=gpt-5.2; v1.0.1", "score": 0.2732, "raw_score": 27.32, "raw_scale": 100, "source_record": "claude-opus-4-6_sierra_2026-05-05", "source_date": "2026-05-06" }, { "id": "44bc6c3b088a15083591", "model": "Gemini 3 Flash", "organization": "Google", "configuration": "Agent: τ-bench; retrieval=terminal; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.2732, "raw_score": 27.32, "raw_scale": 100, "source_record": "gemini-3-flash_sierra_2026-03-02", "source_date": "2026-03-02" }, { "id": "089a2102d75dd0e1767a", "model": "Gemini 3.1 Pro Preview", "organization": "Google", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.26030000000000003, "raw_score": 26.03, "raw_scale": 100, "source_record": "gemini-3-1-pro-preview_sierra_2026-05-05", "source_date": "2026-05-05" }, { "id": "f2359254b77d9b85d446", "model": "Claude Sonnet 4.5", "organization": "Anthropic", "configuration": "Agent: τ-bench; retrieval=terminal; reasoning=enabled; user simulator=gpt-5.2; v1.0.1", "score": 0.2526, "raw_score": 25.26, "raw_scale": 100, "source_record": "claude-sonnet-4-5_sierra_2026-02-26", "source_date": "2026-02-24" }, { "id": "270be9a9a5db557dccdd", "model": "Inkling", "organization": "Thinking Machines Lab", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=max; user simulator=gpt-5.2; v1.0.1", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "inkling_sierra_2026-08-04", "source_date": "2026-07-24" }, { "id": "ac89c8f849dc634168b4", "model": "Claude Opus 4.5", "organization": "Anthropic", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.24739999999999998, "raw_score": 24.74, "raw_scale": 100, "source_record": "claude-opus-4-5_sierra_2026-02-26", "source_date": "2026-05-05" }, { "id": "eade47a9fc1681091049", "model": "Gemini 3 Pro", "organization": "Google", "configuration": "Agent: τ-bench; retrieval=terminal; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.1804, "raw_score": 18.04, "raw_scale": 100, "source_record": "gemini-3-pro_sierra_2026-03-02", "source_date": "2026-03-02" }, { "id": "8d8524fda871df0e192e", "model": "Grok 4.2", "organization": "xAI", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.1804, "raw_score": 18.04, "raw_scale": 100, "source_record": "grok-4-2_sierra_2026-05-05", "source_date": "2026-05-05" }, { "id": "2ac0182606942cec1f78", "model": "Grok 4 fast", "organization": "xAI", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.1572, "raw_score": 15.72, "raw_scale": 100, "source_record": "grok-4-fast_sierra_2026-05-05", "source_date": "2026-05-05" }, { "id": "905c594aa3b2f85f11b0", "model": "Gemini 2.5 Pro", "organization": "Google", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.1366, "raw_score": 13.66, "raw_scale": 100, "source_record": "gemini-2-5-pro_sierra_2026-05-05", "source_date": "2026-05-05" }, { "id": "c4ec5bf93ab2302ed3cf", "model": "Grok 4.1 fast", "organization": "xAI", "configuration": "Agent: τ-bench; retrieval=alltools; reasoning=high; user simulator=gpt-5.2; v1.0.1", "score": 0.13140000000000002, "raw_score": 13.14, "raw_scale": 100, "source_record": "grok-4-1-fast_sierra_2026-05-05", "source_date": "2026-05-05" }, { "id": "c37b3784a5bf790b1f99", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "Agent: τ-bench; retrieval=qwen_embeddings; reasoning=none; user simulator=gpt-5.2; v1.0.1", "score": 0.1263, "raw_score": 12.63, "raw_scale": 100, "source_record": "gpt-5-2-none_sierra_2026-02-26", "source_date": "2026-05-26" }, { "id": "ac7273d8413c0a2c5336", "model": "GLM-5", "organization": "Zhipu AI", "configuration": "Agent: τ-bench; retrieval=text-emb-3-large; reasoning=enabled; user simulator=gpt-5.2; v1.0.1", "score": 0.09789999999999999, "raw_score": 9.79, "raw_scale": 100, "source_record": "glm-5-think_sierra_2026-03-02", "source_date": "2026-02-27" }, { "id": "2500c27bfec2dd766133", "model": "Qwen3.5-397B-A17B", "organization": "Alibaba Cloud", "configuration": "Agent: τ-bench; retrieval=text-emb-3-large; reasoning=enabled; user simulator=gpt-5.2; v1.0.1", "score": 0.09789999999999999, "raw_score": 9.79, "raw_scale": 100, "source_record": "qwen3.5-397b-a17b-think_sierra_2026-03-02", "source_date": "2026-02-27" } ] }, { "id": "workarena-l1", "name": "WorkArena (L1)", "group": "Agent 与工具", "unit": "%", "metric": "Task success", "protocol": { "en": "Official BrowserGym submissions for WorkArena L1. Latest dated result per model/agent/configuration. Submitted protocol changes, including step budgets, remain in result details.", "zh": "官方 BrowserGym 提交，WorkArena L1；各模型、Agent 与配置采用最新日期记录。提交者声明的步骤预算等协议变化保留于成绩详情。" }, "source": { "name": "ServiceNow", "benchmark_creator": "ServiceNow", "url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard", "data_url": "https://huggingface.co/api/spaces/ServiceNow/browsergym-leaderboard/tree/main/results?recursive=true&limit=1000", "sha256": "68dec8d269bf45941c4831f4e71af1943459e7f9caf2d84d217ec144eb0cc436", "retrieved_at": "2026-11-02T03:43:01.414176+00:00", "selector": "results/*/workarena-l1.json; score; latest date_time per agent+configuration", "kind": "official_submissions", "status": "current_snapshot", "artifacts": [ { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/A3-Qwen3.5-9B/workarena-l1.json", "sha256": "c6569b12464c4bd0c00ae31ebed4b4ccedb8b1dcb483464d8ab9879e995be824", "retrieved_at": "2026-11-02T03:44:18.480183+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-AgentTrek-1.0-32b/workarena-l1.json", "sha256": "74a6394fa902cb4b7e18f4d772dfc536d7a611fc53975ed5e3233956907859ae", "retrieved_at": "2026-11-02T03:44:18.342487+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Claude-3.5-Sonnet/workarena-l1.json", "sha256": "a33de4c34c4bd63884c91dcea1c64a551be01330be0897a253fe19960403534d", "retrieved_at": "2026-11-02T03:44:18.369729+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Claude-4-Sonnet/workarena-l1.json", "sha256": "fead207233b57beeb9e0b475d113ba439875537d4d2edd86232f8381366a1ed2", "retrieved_at": "2026-11-02T03:44:19.706108+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-4o-mini/workarena-l1.json", "sha256": "a2135d21629550f960ce9aefcbcb7a7a2b11940471f3a2339722360cbdb173d5", "retrieved_at": "2026-11-02T03:44:19.971291+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-4o/workarena-l1.json", "sha256": "08be6a487c83b24aa52adaae00c53b8e20095be1ed84747280219f3251134670", "retrieved_at": "2026-11-02T03:44:25.165505+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-5-mini/workarena-l1.json", "sha256": "f7c887c8af41cbda87abbf20ccf8c859ad75d7798af524f106bc92b12067ec5e", "retrieved_at": "2026-11-02T03:44:25.215188+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-5-nano/workarena-l1.json", "sha256": "1a66b4656f2781b582fb7284ab7a1103ac23b3f9a7b7258653b40dd050ea901f", "retrieved_at": "2026-11-02T03:44:30.840084+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-5/workarena-l1.json", "sha256": "7ea81bbfd5ce9a8a101bb7d1dea185d514cdb72dee8f2fdbf9a3660a6f8cd2fc", "retrieved_at": "2026-11-02T03:44:30.859197+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-o1-mini/workarena-l1.json", "sha256": "1c75ff75b2ad720a9655f776f7e408b3386a4bea1863bb43de4c063fa61a6584", "retrieved_at": "2026-11-02T03:44:30.959440+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-oss-120b/workarena-l1.json", "sha256": "2bb00d97edb3cdc0a30fcc998fdea19b1f2f9c302a693362e5a2329d5cb38951", "retrieved_at": "2026-11-02T03:44:36.759572+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-oss-20b/workarena-l1.json", "sha256": "cc23870c0a148a840ce4183db58bfd48f7d62ddc031d34d45be7faba19d091d7", "retrieved_at": "2026-11-02T03:44:36.665755+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Llama-3.1-405b/workarena-l1.json", "sha256": "cf57301f5cc66d17bc4b07e2caa40534b4703b3134a5336b035da670026b7a82", "retrieved_at": "2026-11-02T03:45:42.491931+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Llama-3.1-70b/workarena-l1.json", "sha256": "3ec1bc65128a349bfd21370cc8ea276a4f41fcdbc108c242b106a08556260a4f", "retrieved_at": "2026-11-02T03:45:42.471851+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-o3-mini/workarena-l1.json", "sha256": "75e5ab9eab52d9e4aaf97f9cf5856fe5099dff9b1ac9cc9d8d68847d0c83806c", "retrieved_at": "2026-11-02T03:44:43.203753+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/IpaziaHPA-Gemini-3-flash-preview/workarena-l1.json", "sha256": "424cf76d2d3a4ddc619e0c2e27d7fd380032f84662435b9f847630490474c89b", "retrieved_at": "2026-11-02T03:44:43.219223+00:00" } ] }, "results": [ { "id": "da2b4b6630213211d358", "model": "Gemini-3-flash-preview", "organization": "Google", "configuration": "Agent: IpaziaHPA; Increased max_steps from 15 to 30; publisher submission declares a modified evaluation protocol", "score": 0.903, "raw_score": 90.3, "raw_scale": 100, "source_record": "IpaziaHPA-Gemini-3-flash-preview|2026-03-07_00-38-28", "source_date": "2026-03-07 00:38:00", "reported_stderr": 1.6, "protocol_followed": "No", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/IpaziaHPA-Gemini-3-flash-preview/README.md" }, { "id": "e86f2ff72e663a717198", "model": "GPT-5", "organization": "OpenAI", "configuration": "Agent: GenericAgent; Increased max_steps from 15 to 30; publisher submission declares a modified evaluation protocol", "score": 0.7909999999999999, "raw_score": 79.1, "raw_scale": 100, "source_record": "GenericAgent-GPT-5|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 2.2, "protocol_followed": "No", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-5/README.md" }, { "id": "0c3a3e5160c7c684ed64", "model": "Claude-4-Sonnet", "organization": "Anthropic", "configuration": "Agent: GenericAgent", "score": 0.633, "raw_score": 63.3, "raw_scale": 100, "source_record": "GenericAgent-Claude-4-Sonnet|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 2.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Claude-4-Sonnet/README.md" }, { "id": "95526e39bcffdd462a5b", "model": "GPT-5-mini", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.606, "raw_score": 60.6, "raw_scale": 100, "source_record": "GenericAgent-GPT-5-mini|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 2.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-5-mini/README.md" }, { "id": "6eb23f3eef82f47a6652", "model": "Claude-3.5-Sonnet", "organization": "Anthropic", "configuration": "Agent: GenericAgent", "score": 0.564, "raw_score": 56.4, "raw_scale": 100, "source_record": "GenericAgent-Claude-3.5-Sonnet|2024-10-23_14-17-40", "source_date": "2021-01-01 12:00:00", "reported_stderr": 2.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Claude-3.5-Sonnet/README.md" }, { "id": "b1ed12605c7936d115fb", "model": "GPT-o1-mini", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.518, "raw_score": 51.8, "raw_scale": 100, "source_record": "GenericAgent-GPT-o1-mini|f3e1fcb8-5fc5-4115-9e00-27251508e2c7", "source_date": "2025-02-07 14:00:00", "reported_stderr": 2.8, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-o1-mini/README.md" }, { "id": "dd3e48a979b3c8124528", "model": "Qwen3.5-9B", "organization": "Alibaba", "configuration": "Agent: A3; 330 tasks. Fine-tuned on A3-Synth trajectories via the Agent-as-Annotators framework.", "score": 0.515, "raw_score": 51.5, "raw_scale": 100, "source_record": "A3-Qwen3.5-9B|2026-03-12_18-27-40_genericagent-checkpoints-qwen-qwen3-5-9b-web-pro-low-8903051-checkpoint-latest-on-workarena-l1-full", "source_date": "2026-03-12 18:27:40", "reported_stderr": 2.8, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/A3-Qwen3.5-9B/README.md" }, { "id": "f43ab1616f643bd96976", "model": "GPT-oss-120b", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.509, "raw_score": 50.9, "raw_scale": 100, "source_record": "GenericAgent-GPT-oss-120b|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 2.8, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-oss-120b/README.md" }, { "id": "af6c659426320e0850fa", "model": "o3-mini", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.48200000000000004, "raw_score": 48.2, "raw_scale": 100, "source_record": "GenericAgent-o3-mini|f3e1fcb8-5fc5-4115-9e00-27251508e2c7", "source_date": "2025-02-07 14:00:00", "reported_stderr": 2.8, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-o3-mini/README.md" }, { "id": "ddf2478b4126e33bb8e0", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.455, "raw_score": 45.5, "raw_scale": 100, "source_record": "GenericAgent-GPT-4o|2024-10-23_14-17-40", "source_date": "2021-01-01 12:00:00", "reported_stderr": 2.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-4o/README.md" }, { "id": "7d4b1f7f334217cba940", "model": "Llama-3.1-405b", "organization": "Meta", "configuration": "Agent: GenericAgent", "score": 0.433, "raw_score": 43.3, "raw_scale": 100, "source_record": "GenericAgent-Llama-3.1-405b|2024-10-25_17-34-45", "source_date": "2024-10-25 20:32:26", "reported_stderr": 2.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Llama-3.1-405b/README.md" }, { "id": "47caae5a0ccb08a98fd1", "model": "GPT-5-nano", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.406, "raw_score": 40.6, "raw_scale": 100, "source_record": "GenericAgent-GPT-5-nano|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 2.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-5-nano/README.md" }, { "id": "2b2c152fc05ec0101f29", "model": "GPT-oss-20b", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.385, "raw_score": 38.5, "raw_scale": 100, "source_record": "GenericAgent-GPT-oss-20b|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 2.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-oss-20b/README.md" }, { "id": "f49974175bbea2875e8c", "model": "AgentTrek-1.0-32b", "organization": "", "configuration": "Agent: GenericAgent", "score": 0.3829, "raw_score": 38.29, "raw_scale": 100, "source_record": "GenericAgent-AgentTrek-1.0-32b|ed14232c-cd7e-4708-b334-ebaf1f220000", "source_date": "2025-01-12 00:37:04", "reported_stderr": 2.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-AgentTrek-1.0-32b/README.md" }, { "id": "0a13656ad671f986016b", "model": "Llama-3.1-70b", "organization": "Meta", "configuration": "Agent: GenericAgent", "score": 0.27899999999999997, "raw_score": 27.9, "raw_scale": 100, "source_record": "GenericAgent-Llama-3.1-70b|2024-10-23_14-17-40", "source_date": "2024-10-23 22:30:06", "reported_stderr": 2.5, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Llama-3.1-70b/README.md" }, { "id": "433b3143a8c4e0136927", "model": "GPT-4o-mini", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "GenericAgent-GPT-4o-mini|2024-10-23_14-17-40", "source_date": "2021-01-01 12:00:00", "reported_stderr": 2.4, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-4o-mini/README.md" } ] }, { "id": "workarena-l2", "name": "WorkArena++ (L2)", "group": "Agent 与工具", "unit": "%", "metric": "Task success", "protocol": { "en": "Official BrowserGym submissions for WorkArena L2. Latest dated result per model/agent/configuration. Submitted protocol changes, including step budgets, remain in result details.", "zh": "官方 BrowserGym 提交，WorkArena L2；各模型、Agent 与配置采用最新日期记录。提交者声明的步骤预算等协议变化保留于成绩详情。" }, "source": { "name": "ServiceNow", "benchmark_creator": "ServiceNow", "url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard", "data_url": "https://huggingface.co/api/spaces/ServiceNow/browsergym-leaderboard/tree/main/results?recursive=true&limit=1000", "sha256": "68dec8d269bf45941c4831f4e71af1943459e7f9caf2d84d217ec144eb0cc436", "retrieved_at": "2026-11-02T03:43:01.414176+00:00", "selector": "results/*/workarena-l2.json; score; latest date_time per agent+configuration", "kind": "official_submissions", "status": "current_snapshot", "artifacts": [ { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/A3-Qwen3.5-9B/workarena-l2.json", "sha256": "fa7ddf1120884e43aff2cec3480db65d10920de312b71e46cae89e2011c2c95a", "retrieved_at": "2026-11-02T03:44:18.366383+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-AgentTrek-1.0-32b/workarena-l2.json", "sha256": "855fcede12daad9cb6315bdfe5fb2ebaf1af6b730d8d78ac45e2496bb77c0717", "retrieved_at": "2026-11-02T03:44:18.480355+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Claude-3.5-Sonnet/workarena-l2.json", "sha256": "34f9128e7aa9f8e01bae65eaceb94cc2314bb82899e5113b9476ea4c4b510b2e", "retrieved_at": "2026-11-02T03:44:19.692544+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Claude-4-Sonnet/workarena-l2.json", "sha256": "5fb56137adef3ed7c249ecc78642cd7323170b8e9f4dd920395524abad7db39b", "retrieved_at": "2026-11-02T03:44:19.855375+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-4o-mini/workarena-l2.json", "sha256": "a7e7c9f98bbb0a4d8041dbfc046fea06cf67c5efeb6fd4b58098a6c2c6082aa3", "retrieved_at": "2026-11-02T03:44:19.852348+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-4o/workarena-l2.json", "sha256": "1a3be6550ce29c3c8936d13d14be23f9d8c0ca14a42372700579d14a5322f9a6", "retrieved_at": "2026-11-02T03:44:25.154436+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-5-mini/workarena-l2.json", "sha256": "7ca101ac6bb4970547e6188a18ae14d0191f5659e6fad01b2d567d595dead633", "retrieved_at": "2026-11-02T03:44:25.296630+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-5-nano/workarena-l2.json", "sha256": "f72981c3dcd60e2ebb40e590977c980179461fca8ab28300d76924c3ea6635fe", "retrieved_at": "2026-11-02T03:44:30.919765+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-5/workarena-l2.json", "sha256": "c414dfd4b25be3268be8f82bc919416c544e5a36e44ef65c0013a06423499dc4", "retrieved_at": "2026-11-02T03:44:30.824891+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-o1-mini/workarena-l2.json", "sha256": "bc17322f1a54b4339f8f5a79131245b3ea012cf226b5b5caecb54de7774ef518", "retrieved_at": "2026-11-02T03:44:36.680950+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-oss-120b/workarena-l2.json", "sha256": "30f6d49a050903b66b790f76dc63db02f5dd97279234a6aeb891a5ba354795c4", "retrieved_at": "2026-11-02T03:44:36.834769+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-oss-20b/workarena-l2.json", "sha256": "91eb4981a7ae5dc0749deec209a25aa8ba5d9cbb23e931ee212c7804a8d2a72f", "retrieved_at": "2026-11-02T03:44:36.832640+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Llama-3.1-405b/workarena-l2.json", "sha256": "3b4c47354c96735fe3e7b27a209e65d5a6272e99489b3cf81745245e95dc3c21", "retrieved_at": "2026-11-02T03:45:42.482967+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Llama-3.1-70b/workarena-l2.json", "sha256": "8fa434995294d08fdc27e09973e327a28951fe64034550b962a9c30d0d8288f2", "retrieved_at": "2026-11-02T03:44:43.156701+00:00" } ] }, "results": [ { "id": "e86f2ff72e663a717198", "model": "GPT-5", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.6940000000000001, "raw_score": 69.4, "raw_scale": 100, "source_record": "GenericAgent-GPT-5|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 3.0, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-5/README.md" }, { "id": "95526e39bcffdd462a5b", "model": "GPT-5-mini", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.47700000000000004, "raw_score": 47.7, "raw_scale": 100, "source_record": "GenericAgent-GPT-5-mini|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 3.3, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-5-mini/README.md" }, { "id": "0c3a3e5160c7c684ed64", "model": "Claude-4-Sonnet", "organization": "Anthropic", "configuration": "Agent: GenericAgent", "score": 0.40399999999999997, "raw_score": 40.4, "raw_scale": 100, "source_record": "GenericAgent-Claude-4-Sonnet|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 3.2, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Claude-4-Sonnet/README.md" }, { "id": "c0a28fe51dc7af1953b3", "model": "Claude-3.5-Sonnet", "organization": "Anthropic", "configuration": "Agent: GenericAgent", "score": 0.391, "raw_score": 39.1, "raw_scale": 100, "source_record": "GenericAgent-Claude-3.5-Sonnet|2024-10-23_17-10-46", "source_date": "2021-01-01 12:00:00", "reported_stderr": 3.2, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Claude-3.5-Sonnet/README.md" }, { "id": "b8e6ec9d5bede236bf27", "model": "GPT-o1-mini", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "GenericAgent-GPT-o1-mini|2024-10-23_17-10-46", "source_date": "2024-10-24 17:08:53", "reported_stderr": 2.3, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-o1-mini/README.md" }, { "id": "f43ab1616f643bd96976", "model": "GPT-oss-120b", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "GenericAgent-GPT-oss-120b|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 2.1, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-oss-120b/README.md" }, { "id": "a99b781210abe3b15cb1", "model": "Qwen3.5-9B", "organization": "Alibaba", "configuration": "Agent: A3; 341 tasks (full benchmark). Fine-tuned on A3-Synth trajectories via the Agent-as-Annotators framework.", "score": 0.106, "raw_score": 10.6, "raw_scale": 100, "source_record": "A3-Qwen3.5-9B|2026-03-14_12-38-08_genericagent-checkpoints-qwen-qwen3-5-9b-web-pro-low-8903051-checkpoint-latest-on-workarena-l2-test-test", "source_date": "2026-04-06 18:39:04", "reported_stderr": 1.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/A3-Qwen3.5-9B/README.md" }, { "id": "63381480e2d933736305", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.085, "raw_score": 8.5, "raw_scale": 100, "source_record": "GenericAgent-GPT-4o|2024-10-23_17-10-46", "source_date": "2021-01-01 12:00:00", "reported_stderr": 1.8, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-4o/README.md" }, { "id": "770b374a00d7004d6863", "model": "Llama-3.1-405b", "organization": "Meta", "configuration": "Agent: GenericAgent", "score": 0.07200000000000001, "raw_score": 7.2, "raw_scale": 100, "source_record": "GenericAgent-Llama-3.1-405b|528da1f2-1949-41dc-b988-85f19f435af2", "source_date": "2024-11-29 14:28:47", "reported_stderr": 1.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Llama-3.1-405b/README.md" }, { "id": "47caae5a0ccb08a98fd1", "model": "GPT-5-nano", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.034, "raw_score": 3.4, "raw_scale": 100, "source_record": "GenericAgent-GPT-5-nano|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 1.2, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-5-nano/README.md" }, { "id": "42f32e6afb7d2327af77", "model": "AgentTrek-1.0-32b", "organization": "", "configuration": "Agent: GenericAgent", "score": 0.0298, "raw_score": 2.98, "raw_scale": 100, "source_record": "GenericAgent-AgentTrek-1.0-32b|957fb895-8548-46f4-92f0-5de6be7ceb61", "source_date": "2025-01-12 09:39:21", "reported_stderr": 1.1, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-AgentTrek-1.0-32b/README.md" }, { "id": "2b2c152fc05ec0101f29", "model": "GPT-oss-20b", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.026000000000000002, "raw_score": 2.6, "raw_scale": 100, "source_record": "GenericAgent-GPT-oss-20b|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 1.0, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-oss-20b/README.md" }, { "id": "4c9f3b7763842512e794", "model": "Llama-3.1-70b", "organization": "Meta", "configuration": "Agent: GenericAgent", "score": 0.021, "raw_score": 2.1, "raw_scale": 100, "source_record": "GenericAgent-Llama-3.1-70b|2024-10-23_17-10-46", "source_date": "2024-10-24 17:08:53", "reported_stderr": 0.9, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Llama-3.1-70b/README.md" }, { "id": "e1c7037c885e1fa1d341", "model": "GPT-4o-mini", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.013000000000000001, "raw_score": 1.3, "raw_scale": 100, "source_record": "GenericAgent-GPT-4o-mini|2024-10-23_17-10-46", "source_date": "2021-01-01 12:00:00", "reported_stderr": 0.7, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-4o-mini/README.md" } ] }, { "id": "workarena-l3", "name": "WorkArena++ (L3)", "group": "Agent 与工具", "unit": "%", "metric": "Task success", "protocol": { "en": "Official BrowserGym submissions for WorkArena L3. Latest dated result per model/agent/configuration. Submitted protocol changes, including step budgets, remain in result details.", "zh": "官方 BrowserGym 提交，WorkArena L3；各模型、Agent 与配置采用最新日期记录。提交者声明的步骤预算等协议变化保留于成绩详情。" }, "source": { "name": "ServiceNow", "benchmark_creator": "ServiceNow", "url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard", "data_url": "https://huggingface.co/api/spaces/ServiceNow/browsergym-leaderboard/tree/main/results?recursive=true&limit=1000", "sha256": "68dec8d269bf45941c4831f4e71af1943459e7f9caf2d84d217ec144eb0cc436", "retrieved_at": "2026-11-02T03:43:01.414176+00:00", "selector": "results/*/workarena-l3.json; score; latest date_time per agent+configuration", "kind": "official_submissions", "status": "current_snapshot", "artifacts": [ { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-AgentTrek-1.0-32b/workarena-l3.json", "sha256": "a4833791b84344fa444ab93257a568a41ee424a20716146abb7527985882f7fa", "retrieved_at": "2026-11-02T03:44:18.480245+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Claude-3.5-Sonnet/workarena-l3.json", "sha256": "e417a592179b53523cf2b22f31aee050b9fb0ca7ed3735cf6136431c45fdb944", "retrieved_at": "2026-11-02T03:44:19.706203+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-4o-mini/workarena-l3.json", "sha256": "bc5f2f366a091933d81b5cf31d993273dea57fceed327448f297c25e039dd0ee", "retrieved_at": "2026-11-02T03:44:25.174976+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-4o/workarena-l3.json", "sha256": "659f7caaf898e5e08df5d19223c8e850956282ecdb7060f6810df4e2f5fc23b0", "retrieved_at": "2026-11-02T03:44:25.165351+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-5/workarena-l3.json", "sha256": "f448e30ef4d816a4f68692fcfbf12d51ed35c50bff4341d7dab2f88143ea5945", "retrieved_at": "2026-11-02T03:44:30.867794+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-o1-mini/workarena-l3.json", "sha256": "9b9343bc23580734b483f7b645e366c4db6c237b4f34feb9dc272ad13155d6f3", "retrieved_at": "2026-11-02T03:44:36.680905+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Llama-3.1-405b/workarena-l3.json", "sha256": "2a3918313197e5b829547e6f3ee4a7b2b865c0e7b5ac1017d79f4b8ed6638e65", "retrieved_at": "2026-11-02T03:45:42.481284+00:00" }, { "data_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Llama-3.1-70b/workarena-l3.json", "sha256": "856ebe4e16d9d71dab0d66c4d460c58e6abbb751c85da853738426ad50c0cf07", "retrieved_at": "2026-11-02T03:44:43.154443+00:00" } ] }, "results": [ { "id": "e86f2ff72e663a717198", "model": "GPT-5", "organization": "OpenAI", "configuration": "Agent: GenericAgent; Increased max_steps from 50 to 100; publisher submission declares a modified evaluation protocol", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "GenericAgent-GPT-5|2025-08-07_21-09-16", "source_date": "2025-08-07 21:09:16", "reported_stderr": 2.1, "protocol_followed": "No", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-5/README.md" }, { "id": "d868976cb00ed2f0ea49", "model": "Claude-3.5-Sonnet", "organization": "Anthropic", "configuration": "Agent: GenericAgent", "score": 0.004, "raw_score": 0.4, "raw_scale": 100, "source_record": "GenericAgent-Claude-3.5-Sonnet|2024-10-24_18-06-57", "source_date": "2021-01-01 12:00:00", "reported_stderr": 0.4, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Claude-3.5-Sonnet/README.md" }, { "id": "78ff9c9ace169c5f9a7b", "model": "AgentTrek-1.0-32b", "organization": "", "configuration": "Agent: GenericAgent", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "GenericAgent-AgentTrek-1.0-32b|a951b33f-d118-4cf4-a2ef-cc2ef204eeb0", "source_date": "2025-01-13 12:11:45", "reported_stderr": 0.0, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-AgentTrek-1.0-32b/README.md" }, { "id": "4d13cab2fae5bfadc31c", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "GenericAgent-GPT-4o|-", "source_date": "2024-10-24 23:03:30", "reported_stderr": 0.0, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-4o/README.md" }, { "id": "3f2b168d2a4528832838", "model": "GPT-4o-mini", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "GenericAgent-GPT-4o-mini|-", "source_date": "2024-10-24 23:03:30", "reported_stderr": 0.0, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-4o-mini/README.md" }, { "id": "46a3a027286554dddd7f", "model": "GPT-o1-mini", "organization": "OpenAI", "configuration": "Agent: GenericAgent", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "GenericAgent-GPT-o1-mini|-", "source_date": "2024-10-24 23:03:30", "reported_stderr": 0.0, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-GPT-o1-mini/README.md" }, { "id": "8b38fd5f62219332825f", "model": "Llama-3.1-405b", "organization": "Meta", "configuration": "Agent: GenericAgent", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "GenericAgent-Llama-3.1-405b|-", "source_date": "2024-10-24 23:03:30", "reported_stderr": 0.0, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Llama-3.1-405b/README.md" }, { "id": "79e45c52686b69ed2f9e", "model": "Llama-3.1-70b", "organization": "Meta", "configuration": "Agent: GenericAgent", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "GenericAgent-Llama-3.1-70b|-", "source_date": "2024-10-24 23:03:30", "reported_stderr": 0.0, "protocol_followed": "Yes", "report_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/blob/main/results/GenericAgent-Llama-3.1-70b/README.md" } ] }, { "id": "paperbench-baselines", "name": "PaperBench (2025)", "group": "专业工作", "unit": "score", "metric": "Replication score", "protocol": { "en": "Latest published author table contains the 2025-04-02 paper baselines. Default-budget full replication runs; longer-budget and Code-Dev variants are excluded. Mean rubric score on a 0–100 scale.", "zh": "作者目前公开表格仍为 2025-04-02 论文基线。使用默认预算的完整复现成绩；排除更长预算与 Code-Dev。保留 0–100 的平均量规分数。" }, "source": { "data_url": "https://raw.githubusercontent.com/openai/frontier-evals/main/project/paperbench/README.md", "sha256": "b50041ce9329b4a5a779c7114b8c21f6eab93cae91d309f5924f6ebe06f470cb", "retrieved_at": "2026-11-01T16:38:11.207766+00:00", "name": "OpenAI / PaperBench", "url": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "selector": "PaperBench Results: mean score; default-budget rows; exclude 24h/36h and Code-Dev", "status": "paper_baselines", "kind": "official_leaderboard", "benchmark_creator": "OpenAI" }, "results": [ { "id": "990336826d6f366a50c0", "model": "BasicAgent claude-3.5-sonnet", "organization": "Anthropic", "configuration": "Original paper; default budget; 3 runs; mean replication score", "score": 21.0, "raw_score": 21.0, "raw_scale": 1, "source_record": "BasicAgent claude-3.5-sonnet|Original paper; default budget; 3 runs; mean replication score", "source_date": "2025-04-02", "reported_uncertainty": "21.0 ± 0.8" }, { "id": "5b47137c4970c42f7973", "model": "IterativeAgent claude-3.5-sonnet", "organization": "Anthropic", "configuration": "Original paper; default budget; 3 runs; mean replication score", "score": 16.1, "raw_score": 16.1, "raw_scale": 1, "source_record": "IterativeAgent claude-3.5-sonnet|Original paper; default budget; 3 runs; mean replication score", "source_date": "2025-04-02", "reported_uncertainty": "16.1 ± 0.1" }, { "id": "f57a1bf7936b33872121", "model": "BasicAgent o1-high", "organization": "OpenAI", "configuration": "Original paper; default budget; 3 runs; mean replication score", "score": 13.2, "raw_score": 13.2, "raw_scale": 1, "source_record": "BasicAgent o1-high|Original paper; default budget; 3 runs; mean replication score", "source_date": "2025-04-02", "reported_uncertainty": "13.2 ± 0.3" }, { "id": "ba143574b7bdc475aebd", "model": "IterativeAgent o3-mini-high", "organization": "OpenAI", "configuration": "Original paper; default budget; 3 runs; mean replication score", "score": 8.5, "raw_score": 8.5, "raw_scale": 1, "source_record": "IterativeAgent o3-mini-high|Original paper; default budget; 3 runs; mean replication score", "source_date": "2025-04-02", "reported_uncertainty": "8.5 ± 0.8" }, { "id": "7e456f2274a765075120", "model": "BasicAgent deepseek-r1", "organization": "DeepSeek", "configuration": "Original paper; default budget; 3 runs; mean replication score", "score": 6.0, "raw_score": 6.0, "raw_scale": 1, "source_record": "BasicAgent deepseek-r1|Original paper; default budget; 3 runs; mean replication score", "source_date": "2025-04-02", "reported_uncertainty": "6.0 ± 0.3" }, { "id": "0884a95d3624d5f203a0", "model": "BasicAgent gpt-4o", "organization": "OpenAI", "configuration": "Original paper; default budget; 3 runs; mean replication score", "score": 4.1, "raw_score": 4.1, "raw_scale": 1, "source_record": "BasicAgent gpt-4o|Original paper; default budget; 3 runs; mean replication score", "source_date": "2025-04-02", "reported_uncertainty": "4.1 ± 0.1" }, { "id": "c480bef60e984be07198", "model": "BasicAgent gemini-2.0-flash", "organization": "Google", "configuration": "Original paper; default budget; 3 runs; mean replication score", "score": 3.2, "raw_score": 3.2, "raw_scale": 1, "source_record": "BasicAgent gemini-2.0-flash|Original paper; default budget; 3 runs; mean replication score", "source_date": "2025-04-02", "reported_uncertainty": "3.2 ± 0.2" }, { "id": "89e70f896f2f7d17947f", "model": "BasicAgent o3-mini-high", "organization": "OpenAI", "configuration": "Original paper; default budget; 3 runs; mean replication score", "score": 2.6, "raw_score": 2.6, "raw_scale": 1, "source_record": "BasicAgent o3-mini-high|Original paper; default budget; 3 runs; mean replication score", "source_date": "2025-04-02", "reported_uncertainty": "2.6 ± 0.2" } ], "public_history": true }, { "id": "ruler-v1-128k-history", "name": "RULER v1 · 128K (2024–2025)", "group": "指令与长文", "unit": "%", "metric": "128K accuracy", "public_history": true, "protocol": { "en": "Historical 2024–2025 results. Original RULER v1 leaderboard, 128K context column; model sizes/settings remain in the original labels. Distinct from RULER v2.", "zh": "2024–2025 年历史成绩。RULER v1 原榜单的 128K 上下文列；模型规模与设置沿用来源标签。与 RULER v2 分开。" }, "source": { "name": "NVIDIA", "benchmark_creator": "NVIDIA", "url": "https://github.com/NVIDIA/RULER", "data_url": "https://raw.githubusercontent.com/NVIDIA/RULER/ab17b7853df4e0a30b78cd5d2b463ac7dff6ee13/README.md", "sha256": "f3aae5bbd58fcd7de32aab099c455dd2bbc1da23a2efee9db3071eb9fbf894a3", "retrieved_at": "2026-11-02T03:34:36.324321+00:00", "selector": "README main results table: 128K column", "kind": "official_leaderboard", "status": "dated_release" }, "results": [ { "id": "e7040ca360df1ecce043", "model": "Jamba-1.5-large (94B/398B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.951, "raw_score": 95.1, "raw_scale": 100, "source_record": "Jamba-1.5-large (94B/398B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "0cba356934e6b293a8b0", "model": "Gemini-1.5-pro,-Text%20and%20images)", "organization": "Google", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.9440000000000001, "raw_score": 94.4, "raw_scale": 100, "source_record": "Gemini-1.5-pro,-Text%20and%20images)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "498ec6e01e8bbe94f9e4", "model": "Qwen2.5-14B-Instruct-1M (14B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.922, "raw_score": 92.2, "raw_scale": 100, "source_record": "Qwen2.5-14B-Instruct-1M (14B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "ebdf6436a67b39672be5", "model": "Qwen3-235B-A22B (235B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.9059999999999999, "raw_score": 90.6, "raw_scale": 100, "source_record": "Qwen3-235B-A22B (235B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "8420b6e21cbc5fd6bc4b", "model": "Jamba-1.5-mini (12B/52B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.9, "raw_score": 90.0, "raw_scale": 100, "source_record": "Jamba-1.5-mini (12B/52B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "6ab50a8e5625052e2ade", "model": "EXAONE-4.0-32B (32B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.882, "raw_score": 88.2, "raw_scale": 100, "source_record": "EXAONE-4.0-32B (32B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "c27c26e17b7a27edf4f2", "model": "Qwen3-32B (32B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.856, "raw_score": 85.6, "raw_scale": 100, "source_record": "Qwen3-32B (32B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "c4aff122da3c3f714eb5", "model": "Qwen3-14B (14B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.851, "raw_score": 85.1, "raw_scale": 100, "source_record": "Qwen3-14B (14B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "506aee7dfad8a2d55d41", "model": "Qwen2.5-7B-Instruct-1M (7B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.8440000000000001, "raw_score": 84.4, "raw_scale": 100, "source_record": "Qwen2.5-7B-Instruct-1M (7B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "0972664549350dec19ca", "model": "MegaBeam-Mistral (7B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.8370000000000001, "raw_score": 83.7, "raw_scale": 100, "source_record": "MegaBeam-Mistral (7B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "3f97adfab2258483928f", "model": "GLM4 (9B)", "organization": "Z AI", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.831, "raw_score": 83.1, "raw_scale": 100, "source_record": "GLM4 (9B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "25d035d74b1d4136fc6b", "model": "ProLong (8B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.816, "raw_score": 81.6, "raw_scale": 100, "source_record": "ProLong (8B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "621c7e02c31099a129c7", "model": "GPT-4-1106-preview", "organization": "OpenAI", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.812, "raw_score": 81.2, "raw_scale": 100, "source_record": "GPT-4-1106-preview|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "179903f25835417734d9", "model": "Qwen3-30B-A3B (30B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.792, "raw_score": 79.2, "raw_scale": 100, "source_record": "Qwen3-30B-A3B (30B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "00c7fda26eb2503d01aa", "model": "Qwen3-8B (8B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.774, "raw_score": 77.4, "raw_scale": 100, "source_record": "Qwen3-8B (8B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "f80cec57be794e010728", "model": "Yi (34B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.773, "raw_score": 77.3, "raw_scale": 100, "source_record": "Yi (34B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "0b0ef1cc35fd4f2c6c27", "model": "Llama3.1 (8B)", "organization": "Meta", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.77, "raw_score": 77.0, "raw_scale": 100, "source_record": "Llama3.1 (8B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "c5b574f700eb8b860f56", "model": "Command-R (35B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.76, "raw_score": 76.0, "raw_scale": 100, "source_record": "Command-R (35B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "13f77a9ff2eecb641f1a", "model": "Command-R-0824 (32B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.747, "raw_score": 74.7, "raw_scale": 100, "source_record": "Command-R-0824 (32B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "32790a0968e8a0c67708", "model": "GradientAI/Llama3 (70B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.721, "raw_score": 72.1, "raw_scale": 100, "source_record": "GradientAI/Llama3 (70B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "009917fdc3d5a84d2451", "model": "GradientAI/Llama3 (8B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.695, "raw_score": 69.5, "raw_scale": 100, "source_record": "GradientAI/Llama3 (8B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "69d7e15e6c4ee929dec8", "model": "InternLM2.5 (7B)", "organization": "Shanghai AI Laboratory", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.6890000000000001, "raw_score": 68.9, "raw_scale": 100, "source_record": "InternLM2.5 (7B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "a813312d3e2de58a59bc", "model": "Phi3-mini (3.8B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.667, "raw_score": 66.7, "raw_scale": 100, "source_record": "Phi3-mini (3.8B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "4d0d176f3e6a6c4462b1", "model": "Llama3.1 (70B)", "organization": "Meta", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.6659999999999999, "raw_score": 66.6, "raw_scale": 100, "source_record": "Llama3.1 (70B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "ac9d60c72116c149dc58", "model": "Qwen3-4B (4B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.66, "raw_score": 66.0, "raw_scale": 100, "source_record": "Qwen3-4B (4B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "a4ec31d5daa20edd79c2", "model": "LWM (7B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.65, "raw_score": 65.0, "raw_scale": 100, "source_record": "LWM (7B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "a7a4541602ba278e1389", "model": "Command-R-plus-0824 (104B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.6459999999999999, "raw_score": 64.6, "raw_scale": 100, "source_record": "Command-R-plus-0824 (104B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "2311f4d23d20cb0901bb", "model": "Command-R-plus (104B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.631, "raw_score": 63.1, "raw_scale": 100, "source_record": "Command-R-plus (104B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "ba92543de107b113ab25", "model": "Qwen2 (72B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.537, "raw_score": 53.7, "raw_scale": 100, "source_record": "Qwen2 (72B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "9d64274de411de3bb02c", "model": "Mistral-Large-2411 (123B)", "organization": "Mistral AI", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.48100000000000004, "raw_score": 48.1, "raw_scale": 100, "source_record": "Mistral-Large-2411 (123B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "2d6cb0035535c414ae68", "model": "Phi3-medium (14B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.461, "raw_score": 46.1, "raw_scale": 100, "source_record": "Phi3-medium (14B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "4181343f7b7bfad3b935", "model": "Mixtral-8x7B (12.9B/46.7B)", "organization": "Mistral AI", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.445, "raw_score": 44.5, "raw_scale": 100, "source_record": "Mixtral-8x7B (12.9B/46.7B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "d993d29b77fd41dc6796", "model": "GLM3 (6B)", "organization": "Z AI", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.42, "raw_score": 42.0, "raw_scale": 100, "source_record": "GLM3 (6B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "6fa0432345e52bf5db32", "model": "Mixtral-8x22B (39B/141B)", "organization": "Mistral AI", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.317, "raw_score": 31.7, "raw_scale": 100, "source_record": "Mixtral-8x22B (39B/141B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "f1186adcacd61eb64e93", "model": "FILM-7B (7B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.271, "raw_score": 27.1, "raw_scale": 100, "source_record": "FILM-7B (7B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "25445f9b71dcb5159b9b", "model": "Mistral-Large-2407 (123B)", "organization": "Mistral AI", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.237, "raw_score": 23.7, "raw_scale": 100, "source_record": "Mistral-Large-2407 (123B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "4c76830c20da6b354870", "model": "Mistral-Nemo", "organization": "Mistral AI", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.19, "raw_score": 19.0, "raw_scale": 100, "source_record": "Mistral-Nemo|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "497eddd48fefa657c806", "model": "Mistral (7B)", "organization": "Mistral AI", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.138, "raw_score": 13.8, "raw_scale": 100, "source_record": "Mistral (7B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "1873b5f43e46fdec014a", "model": "DBRX (36B/132B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "DBRX (36B/132B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "94d2e671c6c9a5ed9f9a", "model": "LongAlpaca (13B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "LongAlpaca (13B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "588dce89ecd1c21378d1", "model": "LongChat (7B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "LongChat (7B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "8527d280671b2ee36361", "model": "Qwen1.5 (72B)", "organization": "Alibaba", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "Qwen1.5 (72B)|RULER v1; 128K context; 13 synthetic tasks" }, { "id": "9ad59590b8ae1fab4c64", "model": "Together (7B)", "organization": "", "configuration": "RULER v1; 128K context; 13 synthetic tasks", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "Together (7B)|RULER v1; 128K context; 13 synthetic tasks" } ] }, { "id": "u-math-paper", "name": "U-MATH (2024)", "group": "数学与科学", "unit": "%", "metric": "Overall accuracy", "public_history": true, "protocol": { "en": "Historical 2024 results. Paper Table 4, overall U-MATH accuracy; text-only and multimodal model settings remain separate.", "zh": "2024 年历史成绩。论文表 4 的 U-MATH 总体正确率；纯文本与多模态设置分别保留。" }, "source": { "name": "Toloka", "benchmark_creator": "Toloka", "url": "https://github.com/Toloka/u-math", "data_url": "https://arxiv.org/html/2412.03205v3", "sha256": "659a3c4c949110c5a5a166edc9ed1ddcf98828092394224b1ff96dbe3d9c85bb", "retrieved_at": "2026-11-02T03:43:02.166821+00:00", "selector": "Paper Table 4: U-MATH overall, not text-only/visual subcolumns", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "77b9696aec770b92a958", "model": "Gemini-1.5-Pro", "organization": "Google", "configuration": "Paper Table 4; Multimodal models", "score": 0.4889, "raw_score": 48.89, "raw_scale": 100, "source_record": "Gemini-1.5-Pro|Paper Table 4; Multimodal models" }, { "id": "645b3f5d68f45c4dfbcf", "model": "Qwen2.5-Math-72B", "organization": "Alibaba", "configuration": "Paper Table 4; Text-only models", "score": 0.41159999999999997, "raw_score": 41.16, "raw_scale": 100, "source_record": "Qwen2.5-Math-72B|Paper Table 4; Text-only models" }, { "id": "ec1020489d7442d706d1", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Paper Table 4; Multimodal models", "score": 0.3653, "raw_score": 36.53, "raw_scale": 100, "source_record": "GPT-4o|Paper Table 4; Multimodal models" }, { "id": "2214989a3bf3842dbb89", "model": "Qwen2-Math-72B", "organization": "Alibaba", "configuration": "Paper Table 4; Text-only models", "score": 0.35200000000000004, "raw_score": 35.2, "raw_scale": 100, "source_record": "Qwen2-Math-72B|Paper Table 4; Text-only models" }, { "id": "1928119788b30f0c2169", "model": "Qwen2.5-72B", "organization": "Alibaba", "configuration": "Paper Table 4; Text-only models", "score": 0.3404, "raw_score": 34.04, "raw_scale": 100, "source_record": "Qwen2.5-72B|Paper Table 4; Text-only models" }, { "id": "7bd0d82bb590fcadb893", "model": "Qwen2.5-Math-7B", "organization": "Alibaba", "configuration": "Paper Table 4; Text-only models", "score": 0.33240000000000003, "raw_score": 33.24, "raw_scale": 100, "source_record": "Qwen2.5-Math-7B|Paper Table 4; Text-only models" }, { "id": "6ff98822eca782cfca50", "model": "Qwen2.5-7B", "organization": "Alibaba", "configuration": "Paper Table 4; Text-only models", "score": 0.3218, "raw_score": 32.18, "raw_scale": 100, "source_record": "Qwen2.5-7B|Paper Table 4; Text-only models" }, { "id": "4c00c90a2d202c461f18", "model": "Qwen2-Math-7B", "organization": "Alibaba", "configuration": "Paper Table 4; Text-only models", "score": 0.2862, "raw_score": 28.62, "raw_scale": 100, "source_record": "Qwen2-Math-7B|Paper Table 4; Text-only models" }, { "id": "b00a7affe2fab7fc5a05", "model": "Qwen2-VL-72B", "organization": "Alibaba", "configuration": "Paper Table 4; Multimodal models", "score": 0.2693, "raw_score": 26.93, "raw_scale": 100, "source_record": "Qwen2-VL-72B|Paper Table 4; Multimodal models" }, { "id": "ab894b01908d540094c3", "model": "Llama-3.1-70B", "organization": "Meta", "configuration": "Paper Table 4; Text-only models", "score": 0.2613, "raw_score": 26.13, "raw_scale": 100, "source_record": "Llama-3.1-70B|Paper Table 4; Text-only models" }, { "id": "ca0d815e4780ca4f09ca", "model": "Qwen2-VL-7B", "organization": "Alibaba", "configuration": "Paper Table 4; Multimodal models", "score": 0.17329999999999998, "raw_score": 17.33, "raw_scale": 100, "source_record": "Qwen2-VL-7B|Paper Table 4; Multimodal models" }, { "id": "ceb70fd02706448574b3", "model": "Mathstral-7B-v0.1", "organization": "", "configuration": "Paper Table 4; Text-only models", "score": 0.1636, "raw_score": 16.36, "raw_scale": 100, "source_record": "Mathstral-7B-v0.1|Paper Table 4; Text-only models" }, { "id": "d93942b2efb785c7e5a6", "model": "Pixtral-12B-2409", "organization": "", "configuration": "Paper Table 4; Multimodal models", "score": 0.1564, "raw_score": 15.64, "raw_scale": 100, "source_record": "Pixtral-12B-2409|Paper Table 4; Multimodal models" }, { "id": "f657580461ec77212fbf", "model": "Llama-3.1-8B", "organization": "Meta", "configuration": "Paper Table 4; Text-only models", "score": 0.1547, "raw_score": 15.47, "raw_scale": 100, "source_record": "Llama-3.1-8B|Paper Table 4; Text-only models" }, { "id": "c6aee1160e165bbebee8", "model": "LLaVA-OV(Qwen2-7B)", "organization": "", "configuration": "Paper Table 4; Multimodal models", "score": 0.14400000000000002, "raw_score": 14.4, "raw_scale": 100, "source_record": "LLaVA-OV(Qwen2-7B)|Paper Table 4; Multimodal models" } ] }, { "id": "financebench-paper", "name": "FinanceBench (2023)", "group": "专业工作", "unit": "%", "metric": "Reported accuracy", "public_history": true, "protocol": { "en": "Historical 2023 results. Original 150-question human evaluation. Table 2 reports rounded percentages for eight configurations. Figures 2(ii) and 5 give eight more as correct answers out of 150. Long context and oracle use context-first unless marked context-last.", "zh": "2023 年历史成绩。原始 150 题人工评测。表 2 的八种配置保留原表舍入百分比，图 2(ii) 与图 5 的另八种配置按答对题数除以 150。长上下文与 oracle 设置默认将上下文前置，后置设置逐条标明。" }, "source": { "name": "Patronus AI", "benchmark_creator": "Patronus AI", "url": "https://arxiv.org/html/2311.11944v1#S5.T2", "data_url": "https://arxiv.org/html/2311.11944v1", "sha256": "e8b767248f4293ff8b148450e26e9884c1fbab7c09e56acfc42651be10b6ba6b", "retrieved_at": "2026-11-02T05:58:15.117554+00:00", "selector": "Table 2: n=150 rows; Figures 2(ii) and 5: correct-answer series; 16 configurations; excludes Total", "kind": "official_leaderboard", "status": "paper_baselines", "artifacts": [ { "data_url": "https://arxiv.org/html/2311.11944v1/perf_humaneval_prompt_ablation2.svg", "sha256": "a1d82101685dc58d26c6595ee5b93f79df35fd53003212b82c9c27fb10b1ddbb", "retrieved_at": "2026-11-02T05:58:34.969872+00:00" }, { "data_url": "https://arxiv.org/html/2311.11944v1/perf_humaneval_gpt_comparison.svg", "sha256": "57bada2b6ce1b1708cd900f3e37e644b349e261691e7366d69e0f1239c0e9bae", "retrieved_at": "2026-11-02T05:58:35.159165+00:00" } ] }, "results": [ { "id": "c96c57f7f7193ce19356", "model": "GPT-4-Turbo", "organization": "OpenAI", "configuration": "Oracle; context-last; original 150-question panel", "score": 0.8933333333333333, "raw_score": 0.8933333333333333, "raw_scale": 1, "source_record": "Figure 2(ii)|GPT-4-Turbo|Oracle; context-last", "correct_answers": 134, "panel_size": 150, "report_url": "https://arxiv.org/html/2311.11944v1#S5.F2" }, { "id": "bd21be65619ce94a3cda", "model": "GPT-4-Turbo", "organization": "OpenAI", "configuration": "Oracle; original 150-question panel", "score": 0.85, "raw_score": 85.0, "raw_scale": 100, "source_record": "GPT-4-Turbo|Oracle; original 150-question panel", "published_correct_answer": "128 (85%)", "correct_answers": 128, "panel_size": 150 }, { "id": "4c2cc5b42f48186b4962", "model": "GPT-4", "organization": "OpenAI", "configuration": "Oracle; original 150-question panel", "score": 0.84, "raw_score": 0.84, "raw_scale": 1, "source_record": "Figure 5|GPT-4|Oracle", "correct_answers": 126, "panel_size": 150, "report_url": "https://arxiv.org/html/2311.11944v1#A4.F5" }, { "id": "b9c4ea250620c0c45d68", "model": "GPT-4-Turbo", "organization": "OpenAI", "configuration": "Long Context; original 150-question panel", "score": 0.79, "raw_score": 79.0, "raw_scale": 100, "source_record": "GPT-4-Turbo|Long Context; original 150-question panel", "published_correct_answer": "118 (79%)", "correct_answers": 118, "panel_size": 150 }, { "id": "c7dbda5fab0dbc306692", "model": "GPT-4", "organization": "OpenAI", "configuration": "Oracle; context-last; original 150-question panel", "score": 0.7866666666666666, "raw_score": 0.7866666666666666, "raw_scale": 1, "source_record": "Figure 5|GPT-4|Oracle; context-last", "correct_answers": 118, "panel_size": 150, "report_url": "https://arxiv.org/html/2311.11944v1#A4.F5" }, { "id": "ece58e047e44fb924d60", "model": "Claude2", "organization": "Anthropic", "configuration": "Long Context; original 150-question panel", "score": 0.76, "raw_score": 76.0, "raw_scale": 100, "source_record": "Claude2|Long Context; original 150-question panel", "published_correct_answer": "114 (76%)", "correct_answers": 114, "panel_size": 150 }, { "id": "3cdfc29f439c949c4add", "model": "GPT-4-Turbo", "organization": "OpenAI", "configuration": "Single Vector Store; original 150-question panel", "score": 0.5, "raw_score": 50.0, "raw_scale": 100, "source_record": "GPT-4-Turbo|Single Vector Store; original 150-question panel", "published_correct_answer": "75 (50%)", "correct_answers": 75, "panel_size": 150 }, { "id": "5c111000523ad32e91e5", "model": "GPT-4", "organization": "OpenAI", "configuration": "Single Vector Store; original 150-question panel", "score": 0.42, "raw_score": 0.42, "raw_scale": 1, "source_record": "Figure 5|GPT-4|Single Vector Store", "correct_answers": 63, "panel_size": 150, "report_url": "https://arxiv.org/html/2311.11944v1#A4.F5" }, { "id": "6eccbd66a3939b809b1b", "model": "Llama2", "organization": "Meta", "configuration": "Single Vector Store; original 150-question panel", "score": 0.41, "raw_score": 41.0, "raw_scale": 100, "source_record": "Llama2|Single Vector Store; original 150-question panel", "published_correct_answer": "62 (41%)", "correct_answers": 62, "panel_size": 150 }, { "id": "ce78dd09d47f0d5479c9", "model": "Claude2", "organization": "Anthropic", "configuration": "Long Context; context-last; original 150-question panel", "score": 0.37333333333333335, "raw_score": 0.37333333333333335, "raw_scale": 1, "source_record": "Figure 2(ii)|Claude2|Long Context; context-last", "correct_answers": 56, "panel_size": 150, "report_url": "https://arxiv.org/html/2311.11944v1#S5.F2" }, { "id": "5dcd972928530da531cf", "model": "GPT-4-Turbo", "organization": "OpenAI", "configuration": "Long Context; context-last; original 150-question panel", "score": 0.24666666666666667, "raw_score": 0.24666666666666667, "raw_scale": 1, "source_record": "Figure 2(ii)|GPT-4-Turbo|Long Context; context-last", "correct_answers": 37, "panel_size": 150, "report_url": "https://arxiv.org/html/2311.11944v1#S5.F2" }, { "id": "3e2582a07125edc92dca", "model": "GPT-4-Turbo", "organization": "OpenAI", "configuration": "Shared Vector Store; original 150-question panel", "score": 0.19, "raw_score": 19.0, "raw_scale": 100, "source_record": "GPT-4-Turbo|Shared Vector Store; original 150-question panel", "published_correct_answer": "29 (19%)", "correct_answers": 29, "panel_size": 150 }, { "id": "440e5c1a8bd5bdb323ce", "model": "Llama2", "organization": "Meta", "configuration": "Shared Vector Store; original 150-question panel", "score": 0.19, "raw_score": 19.0, "raw_scale": 100, "source_record": "Llama2|Shared Vector Store; original 150-question panel", "published_correct_answer": "29 (19%)", "correct_answers": 29, "panel_size": 150 }, { "id": "b300bf7194eb23d03adc", "model": "GPT-4", "organization": "OpenAI", "configuration": "Shared Vector Store; original 150-question panel", "score": 0.16666666666666666, "raw_score": 0.16666666666666666, "raw_scale": 1, "source_record": "Figure 5|GPT-4|Shared Vector Store", "correct_answers": 25, "panel_size": 150, "report_url": "https://arxiv.org/html/2311.11944v1#A4.F5" }, { "id": "d45f3f0f9fb8bb5035cb", "model": "GPT-4-Turbo", "organization": "OpenAI", "configuration": "Closed Book; original 150-question panel", "score": 0.09, "raw_score": 9.0, "raw_scale": 100, "source_record": "GPT-4-Turbo|Closed Book; original 150-question panel", "published_correct_answer": "14 (9%)", "correct_answers": 14, "panel_size": 150 }, { "id": "bf343aa9117942ba617b", "model": "GPT-4", "organization": "OpenAI", "configuration": "Closed Book; original 150-question panel", "score": 0.04666666666666667, "raw_score": 0.04666666666666667, "raw_scale": 1, "source_record": "Figure 5|GPT-4|Closed Book", "correct_answers": 7, "panel_size": 150, "report_url": "https://arxiv.org/html/2311.11944v1#A4.F5" } ] }, { "id": "crmarena-pro-b2b-single-turn-paper", "name": "CRMArena-Pro · B2B, single-turn (2025)", "group": "专业工作", "unit": "%", "metric": "Overall task success", "public_history": true, "protocol": { "en": "Historical 2025 results. B2B, single-turn; original paper overall column. B2B/B2C and dialogue settings are not pooled.", "zh": "2025 年历史成绩。B2B、单轮；原论文总体列。B2B、B2C 与不同对话设置不混合。" }, "source": { "name": "Salesforce", "benchmark_creator": "Salesforce", "url": "https://github.com/SalesforceAIResearch/CRMArena", "data_url": "https://arxiv.org/html/2505.18878v1", "sha256": "58cf851d7e905b7cd2d8325155b6be386b231cc8a73d972cb3936f411aae36ba", "retrieved_at": "2026-11-02T03:45:49.006738+00:00", "selector": "B2B, single-turn, All column of main performance table", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "c32da3a6a6b8a41f7047", "model": "gemini-2.5-pro", "organization": "Google", "configuration": "Agent: ReAct; B2B; Single-turn Setting", "score": 0.541, "raw_score": 54.1, "raw_scale": 100, "source_record": "gemini-2.5-pro|Agent: ReAct; B2B; Single-turn Setting" }, { "id": "7a3addf61b9da8dae506", "model": "o1", "organization": "OpenAI", "configuration": "Agent: ReAct; B2B; Single-turn Setting", "score": 0.475, "raw_score": 47.5, "raw_scale": 100, "source_record": "o1|Agent: ReAct; B2B; Single-turn Setting" }, { "id": "813abbea9a964fc5caef", "model": "gemini-2.5-flash", "organization": "Google", "configuration": "Agent: ReAct; B2B; Single-turn Setting", "score": 0.419, "raw_score": 41.9, "raw_scale": 100, "source_record": "gemini-2.5-flash|Agent: ReAct; B2B; Single-turn Setting" }, { "id": "f2a4bee82d329a0cee0c", "model": "gemini-2.0-flash", "organization": "Google", "configuration": "Agent: ReAct; B2B; Single-turn Setting", "score": 0.331, "raw_score": 33.1, "raw_scale": 100, "source_record": "gemini-2.0-flash|Agent: ReAct; B2B; Single-turn Setting" }, { "id": "6520c4465e0503d1fcdb", "model": "llama4-maverick", "organization": "Meta", "configuration": "Agent: ReAct; B2B; Single-turn Setting", "score": 0.326, "raw_score": 32.6, "raw_scale": 100, "source_record": "llama4-maverick|Agent: ReAct; B2B; Single-turn Setting" }, { "id": "d30de295faadf6f96eb8", "model": "llama3.1-405b", "organization": "Meta", "configuration": "Agent: ReAct; B2B; Single-turn Setting", "score": 0.287, "raw_score": 28.7, "raw_scale": 100, "source_record": "llama3.1-405b|Agent: ReAct; B2B; Single-turn Setting" }, { "id": "545ca063ec160188d9b0", "model": "gpt-4o", "organization": "OpenAI", "configuration": "Agent: ReAct; B2B; Single-turn Setting", "score": 0.267, "raw_score": 26.7, "raw_scale": 100, "source_record": "gpt-4o|Agent: ReAct; B2B; Single-turn Setting" }, { "id": "cf070d4677a9791fc80b", "model": "llama3.1-70b", "organization": "Meta", "configuration": "Agent: ReAct; B2B; Single-turn Setting", "score": 0.252, "raw_score": 25.2, "raw_scale": 100, "source_record": "llama3.1-70b|Agent: ReAct; B2B; Single-turn Setting" }, { "id": "a381d0e402bce51b1143", "model": "gpt-4o-mini", "organization": "OpenAI", "configuration": "Agent: ReAct; B2B; Single-turn Setting", "score": 0.213, "raw_score": 21.3, "raw_scale": 100, "source_record": "gpt-4o-mini|Agent: ReAct; B2B; Single-turn Setting" } ] }, { "id": "crmarena-pro-b2c-single-turn-paper", "name": "CRMArena-Pro · B2C, single-turn (2025)", "group": "专业工作", "unit": "%", "metric": "Overall task success", "public_history": true, "protocol": { "en": "Historical 2025 results. B2C, single-turn; original paper overall column. B2B/B2C and dialogue settings are not pooled.", "zh": "2025 年历史成绩。B2C、单轮；原论文总体列。B2B、B2C 与不同对话设置不混合。" }, "source": { "name": "Salesforce", "benchmark_creator": "Salesforce", "url": "https://github.com/SalesforceAIResearch/CRMArena", "data_url": "https://arxiv.org/html/2505.18878v1", "sha256": "58cf851d7e905b7cd2d8325155b6be386b231cc8a73d972cb3936f411aae36ba", "retrieved_at": "2026-11-02T03:45:49.006738+00:00", "selector": "B2C, single-turn, All column of main performance table", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "c4898b85bf278de5e3a6", "model": "gemini-2.5-pro", "organization": "Google", "configuration": "Agent: ReAct; B2C; Single-turn Setting", "score": 0.583, "raw_score": 58.3, "raw_scale": 100, "source_record": "gemini-2.5-pro|Agent: ReAct; B2C; Single-turn Setting" }, { "id": "94318d0f713059a9292b", "model": "o1", "organization": "OpenAI", "configuration": "Agent: ReAct; B2C; Single-turn Setting", "score": 0.495, "raw_score": 49.5, "raw_scale": 100, "source_record": "o1|Agent: ReAct; B2C; Single-turn Setting" }, { "id": "68650e91694fed69cca5", "model": "gemini-2.5-flash", "organization": "Google", "configuration": "Agent: ReAct; B2C; Single-turn Setting", "score": 0.467, "raw_score": 46.7, "raw_scale": 100, "source_record": "gemini-2.5-flash|Agent: ReAct; B2C; Single-turn Setting" }, { "id": "2865c0cc3c36a21fd5c0", "model": "llama4-maverick", "organization": "Meta", "configuration": "Agent: ReAct; B2C; Single-turn Setting", "score": 0.375, "raw_score": 37.5, "raw_scale": 100, "source_record": "llama4-maverick|Agent: ReAct; B2C; Single-turn Setting" }, { "id": "fe230957e7c32dcf63de", "model": "gemini-2.0-flash", "organization": "Google", "configuration": "Agent: ReAct; B2C; Single-turn Setting", "score": 0.365, "raw_score": 36.5, "raw_scale": 100, "source_record": "gemini-2.0-flash|Agent: ReAct; B2C; Single-turn Setting" }, { "id": "206a7313cb904e715a5b", "model": "gpt-4o", "organization": "OpenAI", "configuration": "Agent: ReAct; B2C; Single-turn Setting", "score": 0.292, "raw_score": 29.2, "raw_scale": 100, "source_record": "gpt-4o|Agent: ReAct; B2C; Single-turn Setting" }, { "id": "120b2da330a9052b995e", "model": "llama3.1-405b", "organization": "Meta", "configuration": "Agent: ReAct; B2C; Single-turn Setting", "score": 0.265, "raw_score": 26.5, "raw_scale": 100, "source_record": "llama3.1-405b|Agent: ReAct; B2C; Single-turn Setting" }, { "id": "df09823a76b6df467689", "model": "llama3.1-70b", "organization": "Meta", "configuration": "Agent: ReAct; B2C; Single-turn Setting", "score": 0.249, "raw_score": 24.9, "raw_scale": 100, "source_record": "llama3.1-70b|Agent: ReAct; B2C; Single-turn Setting" }, { "id": "28a904538acefd3c4269", "model": "gpt-4o-mini", "organization": "OpenAI", "configuration": "Agent: ReAct; B2C; Single-turn Setting", "score": 0.18600000000000003, "raw_score": 18.6, "raw_scale": 100, "source_record": "gpt-4o-mini|Agent: ReAct; B2C; Single-turn Setting" } ] }, { "id": "crmarena-pro-b2b-multi-turn-paper", "name": "CRMArena-Pro · B2B, multi-turn (2025)", "group": "专业工作", "unit": "%", "metric": "Overall task success", "public_history": true, "protocol": { "en": "Historical 2025 results. B2B, multi-turn; original paper overall column. B2B/B2C and dialogue settings are not pooled.", "zh": "2025 年历史成绩。B2B、多轮；原论文总体列。B2B、B2C 与不同对话设置不混合。" }, "source": { "name": "Salesforce", "benchmark_creator": "Salesforce", "url": "https://github.com/SalesforceAIResearch/CRMArena", "data_url": "https://arxiv.org/html/2505.18878v1", "sha256": "58cf851d7e905b7cd2d8325155b6be386b231cc8a73d972cb3936f411aae36ba", "retrieved_at": "2026-11-02T03:45:49.006738+00:00", "selector": "B2B, multi-turn, All column of main performance table", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "0dbdbe01d21946846d47", "model": "gemini-2.5-pro", "organization": "Google", "configuration": "Agent: ReAct; B2B; Multi-turn Setting", "score": 0.35100000000000003, "raw_score": 35.1, "raw_scale": 100, "source_record": "gemini-2.5-pro|Agent: ReAct; B2B; Multi-turn Setting" }, { "id": "147a84651e5b77d8ea52", "model": "o1", "organization": "OpenAI", "configuration": "Agent: ReAct; B2B; Multi-turn Setting", "score": 0.325, "raw_score": 32.5, "raw_scale": 100, "source_record": "o1|Agent: ReAct; B2B; Multi-turn Setting" }, { "id": "a1dfcbfa637b14937adf", "model": "gemini-2.5-flash", "organization": "Google", "configuration": "Agent: ReAct; B2B; Multi-turn Setting", "score": 0.301, "raw_score": 30.1, "raw_scale": 100, "source_record": "gemini-2.5-flash|Agent: ReAct; B2B; Multi-turn Setting" }, { "id": "9eab026440f491eaa77d", "model": "gemini-2.0-flash", "organization": "Google", "configuration": "Agent: ReAct; B2B; Multi-turn Setting", "score": 0.26899999999999996, "raw_score": 26.9, "raw_scale": 100, "source_record": "gemini-2.0-flash|Agent: ReAct; B2B; Multi-turn Setting" }, { "id": "dc4d43227a38784aa908", "model": "llama4-maverick", "organization": "Meta", "configuration": "Agent: ReAct; B2B; Multi-turn Setting", "score": 0.237, "raw_score": 23.7, "raw_scale": 100, "source_record": "llama4-maverick|Agent: ReAct; B2B; Multi-turn Setting" }, { "id": "8f4ddaee4fca93413653", "model": "llama3.1-405b", "organization": "Meta", "configuration": "Agent: ReAct; B2B; Multi-turn Setting", "score": 0.21, "raw_score": 21.0, "raw_scale": 100, "source_record": "llama3.1-405b|Agent: ReAct; B2B; Multi-turn Setting" }, { "id": "308390525a26e70c81c2", "model": "gpt-4o", "organization": "OpenAI", "configuration": "Agent: ReAct; B2B; Multi-turn Setting", "score": 0.19699999999999998, "raw_score": 19.7, "raw_scale": 100, "source_record": "gpt-4o|Agent: ReAct; B2B; Multi-turn Setting" }, { "id": "2da073303a677acd74ef", "model": "gpt-4o-mini", "organization": "OpenAI", "configuration": "Agent: ReAct; B2B; Multi-turn Setting", "score": 0.193, "raw_score": 19.3, "raw_scale": 100, "source_record": "gpt-4o-mini|Agent: ReAct; B2B; Multi-turn Setting" }, { "id": "b16406a6a0ce028ec17b", "model": "llama3.1-70b", "organization": "Meta", "configuration": "Agent: ReAct; B2B; Multi-turn Setting", "score": 0.163, "raw_score": 16.3, "raw_scale": 100, "source_record": "llama3.1-70b|Agent: ReAct; B2B; Multi-turn Setting" } ] }, { "id": "crmarena-pro-b2c-multi-turn-paper", "name": "CRMArena-Pro · B2C, multi-turn (2025)", "group": "专业工作", "unit": "%", "metric": "Overall task success", "public_history": true, "protocol": { "en": "Historical 2025 results. B2C, multi-turn; original paper overall column. B2B/B2C and dialogue settings are not pooled.", "zh": "2025 年历史成绩。B2C、多轮；原论文总体列。B2B、B2C 与不同对话设置不混合。" }, "source": { "name": "Salesforce", "benchmark_creator": "Salesforce", "url": "https://github.com/SalesforceAIResearch/CRMArena", "data_url": "https://arxiv.org/html/2505.18878v1", "sha256": "58cf851d7e905b7cd2d8325155b6be386b231cc8a73d972cb3936f411aae36ba", "retrieved_at": "2026-11-02T03:45:49.006738+00:00", "selector": "B2C, multi-turn, All column of main performance table", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "e7f81c9745b97adc9f65", "model": "o1", "organization": "OpenAI", "configuration": "Agent: ReAct; B2C; Multi-turn Setting", "score": 0.304, "raw_score": 30.4, "raw_scale": 100, "source_record": "o1|Agent: ReAct; B2C; Multi-turn Setting" }, { "id": "04a1579e990dc943ad69", "model": "gemini-2.5-pro", "organization": "Google", "configuration": "Agent: ReAct; B2C; Multi-turn Setting", "score": 0.3, "raw_score": 30.0, "raw_scale": 100, "source_record": "gemini-2.5-pro|Agent: ReAct; B2C; Multi-turn Setting" }, { "id": "dcf4adcab248f47df088", "model": "gemini-2.0-flash", "organization": "Google", "configuration": "Agent: ReAct; B2C; Multi-turn Setting", "score": 0.23399999999999999, "raw_score": 23.4, "raw_scale": 100, "source_record": "gemini-2.0-flash|Agent: ReAct; B2C; Multi-turn Setting" }, { "id": "42bd51d25e19694fee43", "model": "gemini-2.5-flash", "organization": "Google", "configuration": "Agent: ReAct; B2C; Multi-turn Setting", "score": 0.23399999999999999, "raw_score": 23.4, "raw_scale": 100, "source_record": "gemini-2.5-flash|Agent: ReAct; B2C; Multi-turn Setting" }, { "id": "c06bfbe1e85c9f30622c", "model": "llama3.1-405b", "organization": "Meta", "configuration": "Agent: ReAct; B2C; Multi-turn Setting", "score": 0.2, "raw_score": 20.0, "raw_scale": 100, "source_record": "llama3.1-405b|Agent: ReAct; B2C; Multi-turn Setting" }, { "id": "4ded51fd66627f333c68", "model": "gpt-4o", "organization": "OpenAI", "configuration": "Agent: ReAct; B2C; Multi-turn Setting", "score": 0.183, "raw_score": 18.3, "raw_scale": 100, "source_record": "gpt-4o|Agent: ReAct; B2C; Multi-turn Setting" }, { "id": "f739c430973be4ac5795", "model": "gpt-4o-mini", "organization": "OpenAI", "configuration": "Agent: ReAct; B2C; Multi-turn Setting", "score": 0.158, "raw_score": 15.8, "raw_scale": 100, "source_record": "gpt-4o-mini|Agent: ReAct; B2C; Multi-turn Setting" }, { "id": "ca711ae5b802bf0343c8", "model": "llama3.1-70b", "organization": "Meta", "configuration": "Agent: ReAct; B2C; Multi-turn Setting", "score": 0.151, "raw_score": 15.1, "raw_scale": 100, "source_record": "llama3.1-70b|Agent: ReAct; B2C; Multi-turn Setting" }, { "id": "508c0ff2616f876b3b4f", "model": "llama4-maverick", "organization": "Meta", "configuration": "Agent: ReAct; B2C; Multi-turn Setting", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "llama4-maverick|Agent: ReAct; B2C; Multi-turn Setting" } ] }, { "id": "windows-agent-arena-paper", "name": "Windows Agent Arena (2024)", "group": "Agent 与工具", "unit": "%", "metric": "Task success", "public_history": true, "protocol": { "en": "Historical 2024 results. 154 Windows tasks; Navi with UIA tree + Omniparser. Paper Table 4 Total column; human baseline and Mind2Web results excluded.", "zh": "2024 年历史成绩。154 道 Windows 任务；Navi，UIA tree 与 Omniparser。论文表 4 Total 列；不含人类基线或 Mind2Web 成绩。" }, "source": { "name": "Microsoft", "benchmark_creator": "Microsoft", "url": "https://microsoft.github.io/WindowsAgentArena", "data_url": "https://arxiv.org/html/2409.08264v1", "sha256": "302ffdb97dd53a5cfef8d859128101d4225644dfe6e577694935a1b6a95258d0", "retrieved_at": "2026-11-02T03:51:37.592397+00:00", "selector": "Table 4: last four model rows (UIA + Omniparser), Total", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "cedacd1a4def6dcb84f1", "model": "GPT-4V-1106", "organization": "OpenAI", "configuration": "Agent: Navi; UIA tree + Omniparser; original 154-task paper evaluation", "score": 0.195, "raw_score": 19.5, "raw_scale": 100, "source_record": "GPT-4V-1106|Agent: Navi; UIA tree + Omniparser; original 154-task paper evaluation" }, { "id": "1b03e0c9befdfdb51ed4", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Agent: Navi; UIA tree + Omniparser; original 154-task paper evaluation", "score": 0.086, "raw_score": 8.6, "raw_scale": 100, "source_record": "GPT-4o|Agent: Navi; UIA tree + Omniparser; original 154-task paper evaluation" }, { "id": "dc981cbd9f40337c01cd", "model": "GPT-4o-mini", "organization": "OpenAI", "configuration": "Agent: Navi; UIA tree + Omniparser; original 154-task paper evaluation", "score": 0.042, "raw_score": 4.2, "raw_scale": 100, "source_record": "GPT-4o-mini|Agent: Navi; UIA tree + Omniparser; original 154-task paper evaluation" }, { "id": "7b971e69e7bfc96bcbcc", "model": "Phi3-V", "organization": "", "configuration": "Agent: Navi; UIA tree + Omniparser; original 154-task paper evaluation", "score": 0.035, "raw_score": 3.5, "raw_scale": 100, "source_record": "Phi3-V|Agent: Navi; UIA tree + Omniparser; original 154-task paper evaluation" } ] }, { "id": "mle-bench-2026-04-board", "name": "MLE-bench (2026-04)", "group": "编程", "unit": "%", "metric": "Mean any-medal rate", "public_history": true, "protocol": { "en": "Historical 2026-04 results. Main all-complexity leaderboard, mean any-medal rate. The publisher paused submissions on 2026-04-24. Run budgets remain per result; non-comparable additional submissions are excluded.", "zh": "2026-04 年历史成绩。主榜单全部难度的平均获奖牌率。发布方于 2026-04-24 暂停收录；逐条保留运行预算，不纳入另列的不可比提交。" }, "source": { "name": "OpenAI", "benchmark_creator": "OpenAI", "url": "https://github.com/openai/mle-bench", "data_url": "https://raw.githubusercontent.com/openai/mle-bench/507f92e1138bb6e40dac5c6ee7a6758e6424bf97/README.md", "sha256": "586bbc19cb2a3e11be61021866ef6436cf4b4d935bb1db088984b567b76595bb", "retrieved_at": "2026-11-02T03:38:06.568875+00:00", "selector": "Main leaderboard: All (%) mean; excludes Additional Leaderboard Submissions", "kind": "official_leaderboard", "status": "archived" }, "results": [ { "id": "b7b15f4e5d3a9d2a5da4", "model": "Gemini-3-Pro-Preview", "organization": "Google", "configuration": "Agent: Famou-Agent 2.0; 24 hours; mean any-medal rate", "score": 0.6444, "raw_score": 64.44, "raw_scale": 100, "source_record": "Famou-Agent 2.0|Gemini-3-Pro-Preview|24|2026-02-23", "source_date": "2026-02-23" }, { "id": "67d4829fa93bd608e8fa", "model": "Claude-Opus-4.6", "organization": "Anthropic", "configuration": "Agent: AIBuildAI; 24 hours; mean any-medal rate", "score": 0.6311, "raw_score": 63.11, "raw_scale": 100, "source_record": "AIBuildAI|Claude-Opus-4.6|24|2026-03-06", "source_date": "2026-03-06" }, { "id": "60b65458492d8f0e3497", "model": "Gemini-3-Pro-Preview", "organization": "Google", "configuration": "Agent: CAIR MARS+; 24 hours; mean any-medal rate", "score": 0.6267, "raw_score": 62.67, "raw_scale": 100, "source_record": "CAIR MARS+|Gemini-3-Pro-Preview|24|2026-02-17", "source_date": "2026-02-17" }, { "id": "2c6debb026dff8916ed5", "model": "Gemini-3-Pro-Preview", "organization": "Google", "configuration": "Agent: MLEvolve; 12 hours; mean any-medal rate", "score": 0.6133, "raw_score": 61.33, "raw_scale": 100, "source_record": "MLEvolve|Gemini-3-Pro-Preview|12|2026-02-14", "source_date": "2026-02-14" }, { "id": "4d967baaf0864d88cf62", "model": "Gemini-3-Pro-Preview", "organization": "Google", "configuration": "Agent: PiEvolve (Fractal AI Research); 24 hours; mean any-medal rate", "score": 0.6133, "raw_score": 61.33, "raw_scale": 100, "source_record": "PiEvolve (Fractal AI Research)|Gemini-3-Pro-Preview|24|2026-01-05", "source_date": "2026-01-05" }, { "id": "cf392a50c2b4a24b15f9", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: Famou-Agent 2.0; 24 hours; mean any-medal rate", "score": 0.5956, "raw_score": 59.56, "raw_scale": 100, "source_record": "Famou-Agent 2.0|Gemini-2.5-Pro|24|2025-12-27", "source_date": "2025-12-27" }, { "id": "e89d2d2771d38b95fecd", "model": "Deepseek-V3.2-Speciale", "organization": "DeepSeek", "configuration": "Agent: ML-Master 2.0; 24 hours; mean any-medal rate", "score": 0.5644, "raw_score": 56.44, "raw_scale": 100, "source_record": "ML-Master 2.0|Deepseek-V3.2-Speciale|24|2025-12-16", "source_date": "2025-12-16" }, { "id": "c1dad06b41a21c99e2b4", "model": "Gemini-3-Pro-Preview", "organization": "Google", "configuration": "Agent: CAIR MARS; 24 hours; mean any-medal rate", "score": 0.56, "raw_score": 56.0, "raw_scale": 100, "source_record": "CAIR MARS|Gemini-3-Pro-Preview|24|2026-01-25", "source_date": "2026-01-25" }, { "id": "4aa9a7c9c56107f235bc", "model": "Gemini-3-Pro-Preview", "organization": "Google", "configuration": "Agent: PiEvolve (Fractal AI Research); 12 hours; mean any-medal rate", "score": 0.52, "raw_score": 52.0, "raw_scale": 100, "source_record": "PiEvolve (Fractal AI Research)|Gemini-3-Pro-Preview|12|2026-01-05", "source_date": "2026-01-05" }, { "id": "32987793c1695cc431f0", "model": "Gemini-3-Pro-Preview", "organization": "Google", "configuration": "Agent: Leeroo; 24 hours; mean any-medal rate", "score": 0.5067, "raw_score": 50.67, "raw_scale": 100, "source_record": "Leeroo|Gemini-3-Pro-Preview|24|2025-12-07", "source_date": "2025-12-07" }, { "id": "812101ca58a3aaa05fec", "model": "gpt-5-codex", "organization": "OpenAI", "configuration": "Agent: Thesis; 24 hours; mean any-medal rate", "score": 0.4844, "raw_score": 48.44, "raw_scale": 100, "source_record": "Thesis|gpt-5-codex|24|2025-11-10", "source_date": "2025-11-10" }, { "id": "82b2c8621cd02df9aee2", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: CAIR MLE-STAR-Pro-1.5; 24 hours; mean any-medal rate", "score": 0.44, "raw_score": 44.0, "raw_scale": 100, "source_record": "CAIR MLE-STAR-Pro-1.5|Gemini-2.5-Pro|24|2025-11-25", "source_date": "2025-11-25" }, { "id": "bc53f60a5b7a2fd26d69", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: Famou-Agent; 24 hours; mean any-medal rate", "score": 0.43560000000000004, "raw_score": 43.56, "raw_scale": 100, "source_record": "Famou-Agent|Gemini-2.5-Pro|24|2025-10-10", "source_date": "2025-10-10" }, { "id": "c7463d556c9aefb59f1e", "model": "gpt-5 (low verbosity/effort)", "organization": "OpenAI", "configuration": "Agent: Operand ensemble; 24 hours; mean any-medal rate", "score": 0.3956, "raw_score": 39.56, "raw_scale": 100, "source_record": "Operand ensemble|gpt-5 (low verbosity/effort)|24|2025-10-06", "source_date": "2025-10-06" }, { "id": "bba7bc2f3827bb5f8dd8", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Agent: CAIR MLE-STAR-Pro-1.0; 12 hours; mean any-medal rate", "score": 0.38670000000000004, "raw_score": 38.67, "raw_scale": 100, "source_record": "CAIR MLE-STAR-Pro-1.0|Gemini-2.5-Pro|12|2025-11-03", "source_date": "2025-11-03" }, { "id": "d0ee6de7afeda6ada0ac", "model": "deepseek-r1", "organization": "DeepSeek", "configuration": "Agent: InternAgent; 12 hours; mean any-medal rate", "score": 0.3644, "raw_score": 36.44, "raw_scale": 100, "source_record": "InternAgent|deepseek-r1|12|2025-09-12", "source_date": "2025-09-12" }, { "id": "a7099ac03d6e1347d69f", "model": "gpt-5", "organization": "OpenAI", "configuration": "Agent: R&D-Agent; 12 hours; mean any-medal rate", "score": 0.35109999999999997, "raw_score": 35.11, "raw_scale": 100, "source_record": "R&D-Agent|gpt-5|12|2025-09-26", "source_date": "2025-09-26" }, { "id": "ad91e4dad6c707d0b662", "model": "undisclosed", "organization": "", "configuration": "Agent: Neo multi-agent; 36 hours; mean any-medal rate", "score": 0.3422, "raw_score": 34.22, "raw_scale": 100, "source_record": "Neo multi-agent|undisclosed|36|2025-07-28", "source_date": "2025-07-28" }, { "id": "2d5e81dac66dcc8a1b05", "model": "o3", "organization": "OpenAI", "configuration": "Agent: AIRA-dojo; 24 hours; mean any-medal rate", "score": 0.316, "raw_score": 31.6, "raw_scale": 100, "source_record": "AIRA-dojo|o3|24|2025-05-15", "source_date": "2025-05-15" }, { "id": "b5c37615030fec287690", "model": "o3 + GPT-4.1", "organization": "", "configuration": "Agent: R&D-Agent; 24 hours; mean any-medal rate", "score": 0.30219999999999997, "raw_score": 30.22, "raw_scale": 100, "source_record": "R&D-Agent|o3 + GPT-4.1|24|2025-08-15", "source_date": "2025-08-15" }, { "id": "190beb41505071de5720", "model": "deepseek-r1", "organization": "DeepSeek", "configuration": "Agent: ML-Master; 12 hours; mean any-medal rate", "score": 0.2933, "raw_score": 29.33, "raw_scale": 100, "source_record": "ML-Master|deepseek-r1|12|2025-06-17", "source_date": "2025-06-17" }, { "id": "889f4362127f77ead306", "model": "o1-preview", "organization": "OpenAI", "configuration": "Agent: R&D-Agent; 24 hours; mean any-medal rate", "score": 0.22399999999999998, "raw_score": 22.4, "raw_scale": 100, "source_record": "R&D-Agent|o1-preview|24|2025-05-14", "source_date": "2025-05-14" }, { "id": "387af4f3234cafcfab70", "model": "o1-preview", "organization": "OpenAI", "configuration": "Agent: AIDE; 24 hours; mean any-medal rate", "score": 0.17120000000000002, "raw_score": 17.12, "raw_scale": 100, "source_record": "AIDE|o1-preview|24|2024-10-08", "source_date": "2024-10-08" }, { "id": "65af4dbe5c78e3a0431f", "model": "gpt-4o-2024-08-06", "organization": "OpenAI", "configuration": "Agent: AIDE; 24 hours; mean any-medal rate", "score": 0.0863, "raw_score": 8.63, "raw_scale": 100, "source_record": "AIDE|gpt-4o-2024-08-06|24|2024-10-08", "source_date": "2024-10-08" }, { "id": "08cc4bc687b973c1473a", "model": "claude-3-5-sonnet-20240620", "organization": "Anthropic", "configuration": "Agent: AIDE; 24 hours; mean any-medal rate", "score": 0.0756, "raw_score": 7.56, "raw_scale": 100, "source_record": "AIDE|claude-3-5-sonnet-20240620|24|2024-10-08", "source_date": "2024-10-08" }, { "id": "9f2d94d240601c98d7d5", "model": "gpt-4o-2024-08-06", "organization": "OpenAI", "configuration": "Agent: OpenHands; 24 hours; mean any-medal rate", "score": 0.0489, "raw_score": 4.89, "raw_scale": 100, "source_record": "OpenHands|gpt-4o-2024-08-06|24|2024-10-08", "source_date": "2024-10-08" }, { "id": "9dfe9f5e62d66a5d6965", "model": "llama-3.1-405b-instruct", "organization": "Meta", "configuration": "Agent: AIDE; 24 hours; mean any-medal rate", "score": 0.0333, "raw_score": 3.33, "raw_scale": 100, "source_record": "AIDE|llama-3.1-405b-instruct|24|2024-10-08", "source_date": "2024-10-08" }, { "id": "54f6d913d3ae1b1500ce", "model": "gpt-4o-2024-08-06", "organization": "OpenAI", "configuration": "Agent: MLAB; 24 hours; mean any-medal rate", "score": 0.016, "raw_score": 1.6, "raw_scale": 100, "source_record": "MLAB|gpt-4o-2024-08-06|24|2024-10-08", "source_date": "2024-10-08" } ] }, { "id": "swe-lancer-diamond-paper", "name": "SWE-Lancer Diamond (2025)", "group": "编程", "unit": "%", "metric": "Pass@1", "public_history": true, "protocol": { "en": "Historical 2025 results. Original 502-task Diamond set, IC + manager tasks, paper v4 Tables 1 and 11 pass@1. The two open-weight runs use the same scaffold, with one seed and preliminary status. The July 198-task offline IC release is separate.", "zh": "2025 年历史成绩。原始 502 题 Diamond 集，IC 与管理任务合计，论文 v4 表 1 与表 11 的 pass@1。两项开放权重模型沿用相同框架，各运行一个种子，属初步结果。7 月的 198 题离线 IC 另为一批。" }, "source": { "name": "OpenAI", "benchmark_creator": "OpenAI", "url": "https://arxiv.org/html/2502.12115v4", "data_url": "https://arxiv.org/html/2502.12115v4", "sha256": "f38ac737a3ca870c4edca1189569b804bfcbd2d58964009938135ad5d1e5355a", "retrieved_at": "2026-11-02T05:58:29.594400+00:00", "selector": "Paper v4 Tables 1 and 11: Dataset=SWE-Lancer Diamond, pass@1; excludes earnings and IC-only rows", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "a171fb517911999f40de", "model": "3.5 Sonnet", "organization": "", "configuration": "Diamond; reasoning=N/A; original paper scaffold", "score": 0.361, "raw_score": 36.1, "raw_scale": 100, "source_record": "3.5 Sonnet|Diamond; reasoning=N/A; original paper scaffold" }, { "id": "0a4c775c1442418d9c55", "model": "o1", "organization": "OpenAI", "configuration": "Diamond; reasoning=High; original paper scaffold", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "o1|Diamond; reasoning=High; original paper scaffold" }, { "id": "34797b849e1fb2f4f2b9", "model": "Llama 3.3 70B", "organization": "Meta", "configuration": "Diamond; original paper scaffold; preliminary one-seed run", "score": 0.235, "raw_score": 23.5, "raw_scale": 100, "source_record": "Llama 3.3 70B|Diamond; original paper scaffold; preliminary one-seed run", "report_url": "https://arxiv.org/html/2502.12115v4#A1.T11" }, { "id": "faaca17bb68d8ca89fc3", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Diamond; reasoning=N/A; original paper scaffold", "score": 0.233, "raw_score": 23.3, "raw_scale": 100, "source_record": "GPT-4o|Diamond; reasoning=N/A; original paper scaffold" }, { "id": "42f9e88cc5b6860514a9", "model": "Deepseek-R1", "organization": "DeepSeek", "configuration": "Diamond; original paper scaffold; preliminary one-seed run", "score": 0.231, "raw_score": 23.1, "raw_scale": 100, "source_record": "Deepseek-R1|Diamond; original paper scaffold; preliminary one-seed run", "report_url": "https://arxiv.org/html/2502.12115v4#A1.T11" } ] }, { "id": "swe-lancer-full-paper", "name": "SWE-Lancer Full (2025)", "group": "编程", "unit": "%", "metric": "Pass@1", "public_history": true, "protocol": { "en": "Historical 2025 results. Original 1488-task Full set, IC + manager tasks, paper v4 Tables 1 and 11 pass@1. The two open-weight runs use the same scaffold, with one seed and preliminary status. The July 198-task offline IC release is separate.", "zh": "2025 年历史成绩。原始 1488 题 Full 集，IC 与管理任务合计，论文 v4 表 1 与表 11 的 pass@1。两项开放权重模型沿用相同框架，各运行一个种子，属初步结果。7 月的 198 题离线 IC 另为一批。" }, "source": { "name": "OpenAI", "benchmark_creator": "OpenAI", "url": "https://arxiv.org/html/2502.12115v4", "data_url": "https://arxiv.org/html/2502.12115v4", "sha256": "f38ac737a3ca870c4edca1189569b804bfcbd2d58964009938135ad5d1e5355a", "retrieved_at": "2026-11-02T05:58:29.594400+00:00", "selector": "Paper v4 Tables 1 and 11: Dataset=SWE-Lancer Full, pass@1; excludes earnings and IC-only rows", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "8d7a69e37db011c54524", "model": "3.5 Sonnet", "organization": "", "configuration": "Full; reasoning=N/A; original paper scaffold", "score": 0.337, "raw_score": 33.7, "raw_scale": 100, "source_record": "3.5 Sonnet|Full; reasoning=N/A; original paper scaffold" }, { "id": "08ef1923cb10b7183f63", "model": "o1", "organization": "OpenAI", "configuration": "Full; reasoning=High; original paper scaffold", "score": 0.32899999999999996, "raw_score": 32.9, "raw_scale": 100, "source_record": "o1|Full; reasoning=High; original paper scaffold" }, { "id": "badd0e49646dcd2c95ff", "model": "Deepseek-R1", "organization": "DeepSeek", "configuration": "Full; original paper scaffold; preliminary one-seed run", "score": 0.254, "raw_score": 25.4, "raw_scale": 100, "source_record": "Deepseek-R1|Full; original paper scaffold; preliminary one-seed run", "report_url": "https://arxiv.org/html/2502.12115v4#A1.T11" }, { "id": "b8308bcc1c462180eaa2", "model": "Llama 3.3 70B", "organization": "Meta", "configuration": "Full; original paper scaffold; preliminary one-seed run", "score": 0.245, "raw_score": 24.5, "raw_scale": 100, "source_record": "Llama 3.3 70B|Full; original paper scaffold; preliminary one-seed run", "report_url": "https://arxiv.org/html/2502.12115v4#A1.T11" }, { "id": "d76c563bc3f4b7cff46c", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Full; reasoning=N/A; original paper scaffold", "score": 0.233, "raw_score": 23.3, "raw_scale": 100, "source_record": "GPT-4o|Full; reasoning=N/A; original paper scaffold" } ] }, { "id": "clawsbench-openclaw-paper", "name": "ClawsBench (2026-04)", "group": "Agent 与工具", "unit": "%", "metric": "Non-safety task success rate", "public_history": true, "protocol": { "en": "Historical 2026-04 results. Paper Table 4: OpenClaw, domain skills and meta prompt both on. Non-safety task success requires task score ≥0.8; this is TSR, not unsafe-action rate or safe-completion rate.", "zh": "2026-04 年历史成绩。论文表 4：OpenClaw，领域 Skills 与元提示均开启。非安全任务得分 ≥0.8 计成功；使用 TSR，不使用不安全动作率或安全任务完成率。" }, "source": { "name": "BenchFlow", "benchmark_creator": "BenchFlow", "url": "https://benchflow-ai.github.io/ClawsBench/", "data_url": "https://arxiv.org/html/2604.05172v1", "sha256": "8eed566db916ee262b46545e1a4273a075b0d7d79ceea7d3b911ff6a62e3dd07", "retrieved_at": "2026-11-02T03:53:46.119452+00:00", "selector": "Paper Table 4: Sk=✓, Mt=✓; TSR column, excluding confidence intervals", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "a033dfb122760445dbef", "model": "Claude Opus 4.6", "organization": "Anthropic", "configuration": "Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "score": 0.63, "raw_score": 63.0, "raw_scale": 100, "source_record": "Claude Opus 4.6|Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "reported_interval": "[42,82]" }, { "id": "2a86601a413a3402a312", "model": "GLM-5", "organization": "Z AI", "configuration": "Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "score": 0.6, "raw_score": 60.0, "raw_scale": 100, "source_record": "GLM-5|Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "reported_interval": "[41,78]" }, { "id": "b2efbecff2a6a4d0635d", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "score": 0.58, "raw_score": 58.0, "raw_scale": 100, "source_record": "Gemini 3.1 Pro|Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "reported_interval": "[38,77]" }, { "id": "76f97c232261a5769819", "model": "Claude Sonnet 4.6", "organization": "Anthropic", "configuration": "Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "score": 0.56, "raw_score": 56.0, "raw_scale": 100, "source_record": "Claude Sonnet 4.6|Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "reported_interval": "[36,75]" }, { "id": "609db0e5c57e59339a5d", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "score": 0.53, "raw_score": 53.0, "raw_scale": 100, "source_record": "GPT-5.4|Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "reported_interval": "[34,72]" }, { "id": "3d871e11f5d2467f1e2a", "model": "Gemini 3.1 Flash-Lite", "organization": "Google", "configuration": "Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "score": 0.39, "raw_score": 39.0, "raw_scale": 100, "source_record": "Gemini 3.1 Flash-Lite|Agent: OpenClaw; domain skills=on; meta prompt=on; non-safety tasks", "reported_interval": "[21,57]" } ] }, { "id": "saas-bench-paper-resolved", "name": "SaaS-Bench (Resolved, May 2026 paper)", "group": "Agent 与工具", "unit": "%", "metric": "Resolved", "source": { "data_url": "https://arxiv.org/html/2605.15777v2", "sha256": "7e4a5d77d6f88f23b2c179726b5b023d4f287eb30fe64c14f4bc209452de569c", "retrieved_at": "2026-11-02T05:58:02.656686+00:00", "name": "UniPat AI", "url": "https://arxiv.org/html/2605.15777v2#S4.T2", "selector": "Figure 1 and Table 2. Resolved for the task set evaluated by each model.", "kind": "official_leaderboard", "status": "paper_baselines" }, "protocol": { "en": "Paper Figure 1, Browser-Use. Eleven models ran all 106 tasks. The three marked text-only ran 74 tasks. A task counts as resolved when every checkpoint passes.", "zh": "论文图 1 使用 Browser-Use。11 个模型评测全部 106 道任务，标有「仅文本」的 3 个模型评测 74 道纯文本任务。任务的所有检查点均通过才算完成。" }, "results": [ { "id": "e80ba415aa4f98ac5635", "model": "Claude Opus 4.7", "organization": "Anthropic", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.038, "raw_score": 3.8, "raw_scale": 100, "source_record": "Table 2|Claude Opus 4.7|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.3" }, { "id": "754116a4b2f448ffa6eb", "model": "GPT-5.4 High", "organization": "OpenAI", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.038, "raw_score": 3.8, "raw_scale": 100, "source_record": "Table 2|GPT-5.4 High|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.6" }, { "id": "2d7925c58f8fa70f6b42", "model": "Claude Opus 4.6", "organization": "Anthropic", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.019, "raw_score": 1.9, "raw_scale": 100, "source_record": "Table 2|Claude Opus 4.6|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.5" }, { "id": "3b4d547a789215023810", "model": "Doubao Seed 2.0 Pro", "organization": "ByteDance", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.019, "raw_score": 1.9, "raw_scale": 100, "source_record": "Table 2|Doubao Seed 2.0 Pro|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.11" }, { "id": "38fcb7b11a324b9e0849", "model": "GPT-5.5 High", "organization": "OpenAI", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.019, "raw_score": 1.9, "raw_scale": 100, "source_record": "Table 2|GPT-5.5 High|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.4" }, { "id": "c2e9822e0c74132be6b0", "model": "Qwen 3.6 Plus", "organization": "Alibaba", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.019, "raw_score": 1.9, "raw_scale": 100, "source_record": "Table 2|Qwen 3.6 Plus|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.8" }, { "id": "aa3464eaa81e84c81d8a", "model": "DeepSeek V4 Pro", "organization": "DeepSeek", "configuration": "Browser-Use; May 2026 paper; 74 text-only tasks", "score": 0.013999999999999999, "raw_score": 1.4, "raw_scale": 100, "source_record": "Table 2|DeepSeek V4 Pro|Resolved", "source_column": "Resolved", "evaluation_scope": "text_only", "task_count": 74, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.14" }, { "id": "a6b4c83c6521aa0c6a7b", "model": "Claude Sonnet 4.6", "organization": "Anthropic", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.009000000000000001, "raw_score": 0.9, "raw_scale": 100, "source_record": "Table 2|Claude Sonnet 4.6|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.13" }, { "id": "bf6d861ae601c476bc34", "model": "Gemini 3.5 Flash High", "organization": "Google", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.009000000000000001, "raw_score": 0.9, "raw_scale": 100, "source_record": "Table 2|Gemini 3.5 Flash High|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.12" }, { "id": "8d0da1463fbef839a8e8", "model": "Kimi K2.6", "organization": "Moonshot AI", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.009000000000000001, "raw_score": 0.9, "raw_scale": 100, "source_record": "Table 2|Kimi K2.6|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.7" }, { "id": "f4a9972c6567c3a180b3", "model": "GLM-5.1", "organization": "Z AI", "configuration": "Browser-Use; May 2026 paper; 74 text-only tasks", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "Table 2|GLM-5.1|Resolved", "source_column": "Resolved", "evaluation_scope": "text_only", "task_count": 74, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.15" }, { "id": "1883789f85c34ba075b5", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "Table 2|Gemini 3.1 Pro|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.10" }, { "id": "74f83bd9958df90da421", "model": "Kimi K2.5", "organization": "Moonshot AI", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "Table 2|Kimi K2.5|Resolved", "source_column": "Resolved", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.9" }, { "id": "3b33692f5238642ba5bd", "model": "MiniMax M2.7", "organization": "MiniMax", "configuration": "Browser-Use; May 2026 paper; 74 text-only tasks", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "Table 2|MiniMax M2.7|Resolved", "source_column": "Resolved", "evaluation_scope": "text_only", "task_count": 74, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.16" } ], "public_history": true }, { "id": "saas-bench-paper-checkpoint", "name": "SaaS-Bench (Checkpoint, May 2026 paper)", "group": "Agent 与工具", "unit": "score", "metric": "Checkpoint", "source": { "data_url": "https://arxiv.org/html/2605.15777v2", "sha256": "7e4a5d77d6f88f23b2c179726b5b023d4f287eb30fe64c14f4bc209452de569c", "retrieved_at": "2026-11-02T05:58:02.656686+00:00", "name": "UniPat AI", "url": "https://arxiv.org/html/2605.15777v2#S4.T2", "selector": "Figure 1 and Table 2. Overall for full runs, Text-Only / Overall for daggered runs.", "kind": "official_leaderboard", "status": "paper_baselines" }, "protocol": { "en": "Paper Figure 1, Browser-Use. Eleven models ran all 106 tasks. The three marked text-only ran 74 tasks. Scores are weighted checkpoint credit on a 0–100 scale.", "zh": "论文图 1 使用 Browser-Use。11 个模型评测全部 106 道任务，标有「仅文本」的 3 个模型评测 74 道纯文本任务。检查点按权重计分，满分 100。" }, "results": [ { "id": "b13ba0b083d0146b9fc6", "model": "Claude Opus 4.7", "organization": "Anthropic", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 43.9, "raw_score": 43.9, "raw_scale": 1, "source_record": "Table 2|Claude Opus 4.7|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.3" }, { "id": "a86c68471c3188447ce4", "model": "GPT-5.5 High", "organization": "OpenAI", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 43.8, "raw_score": 43.8, "raw_scale": 1, "source_record": "Table 2|GPT-5.5 High|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.4" }, { "id": "24a69965a2f67b1b735e", "model": "Claude Opus 4.6", "organization": "Anthropic", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 43.2, "raw_score": 43.2, "raw_scale": 1, "source_record": "Table 2|Claude Opus 4.6|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.5" }, { "id": "fa60c66956774cf6c205", "model": "GPT-5.4 High", "organization": "OpenAI", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 37.0, "raw_score": 37.0, "raw_scale": 1, "source_record": "Table 2|GPT-5.4 High|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.6" }, { "id": "c350a7f4bbf2cb96b131", "model": "Kimi K2.6", "organization": "Moonshot AI", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 34.1, "raw_score": 34.1, "raw_scale": 1, "source_record": "Table 2|Kimi K2.6|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.7" }, { "id": "9264159f112610a850f3", "model": "Qwen 3.6 Plus", "organization": "Alibaba", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 29.9, "raw_score": 29.9, "raw_scale": 1, "source_record": "Table 2|Qwen 3.6 Plus|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.8" }, { "id": "d4fb726b5c21ae9e8630", "model": "Kimi K2.5", "organization": "Moonshot AI", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 27.7, "raw_score": 27.7, "raw_scale": 1, "source_record": "Table 2|Kimi K2.5|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.9" }, { "id": "1772fa02ce57b62899cd", "model": "Doubao Seed 2.0 Pro", "organization": "ByteDance", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 27.1, "raw_score": 27.1, "raw_scale": 1, "source_record": "Table 2|Doubao Seed 2.0 Pro|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.11" }, { "id": "5318f805078ce2db6e69", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 27.1, "raw_score": 27.1, "raw_scale": 1, "source_record": "Table 2|Gemini 3.1 Pro|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.10" }, { "id": "f59377643a0693799259", "model": "Claude Sonnet 4.6", "organization": "Anthropic", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 23.3, "raw_score": 23.3, "raw_scale": 1, "source_record": "Table 2|Claude Sonnet 4.6|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.13" }, { "id": "7423e0f618cae68e4b17", "model": "Gemini 3.5 Flash High", "organization": "Google", "configuration": "Browser-Use; May 2026 paper; full 106-task set", "score": 23.3, "raw_score": 23.3, "raw_scale": 1, "source_record": "Table 2|Gemini 3.5 Flash High|Overall", "source_column": "Overall", "evaluation_scope": "full", "task_count": 106, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.12" }, { "id": "acf0115a0ca1a1da0003", "model": "DeepSeek V4 Pro", "organization": "DeepSeek", "configuration": "Browser-Use; May 2026 paper; 74 text-only tasks", "score": 21.5, "raw_score": 21.5, "raw_scale": 1, "source_record": "Table 2|DeepSeek V4 Pro|Text-Only / Overall", "source_column": "Text-Only / Overall", "evaluation_scope": "text_only", "task_count": 74, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.14" }, { "id": "d50bc577e3a7cbe9175e", "model": "GLM-5.1", "organization": "Z AI", "configuration": "Browser-Use; May 2026 paper; 74 text-only tasks", "score": 17.4, "raw_score": 17.4, "raw_scale": 1, "source_record": "Table 2|GLM-5.1|Text-Only / Overall", "source_column": "Text-Only / Overall", "evaluation_scope": "text_only", "task_count": 74, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.15" }, { "id": "d127a831218eb4343109", "model": "MiniMax M2.7", "organization": "MiniMax", "configuration": "Browser-Use; May 2026 paper; 74 text-only tasks", "score": 15.8, "raw_score": 15.8, "raw_scale": 1, "source_record": "Table 2|MiniMax M2.7|Text-Only / Overall", "source_column": "Text-Only / Overall", "evaluation_scope": "text_only", "task_count": 74, "report_url": "https://arxiv.org/html/2605.15777v2#S4.T2.6.1.16" } ], "public_history": true, "score_max": 100 }, { "id": "osworld-2-20260624-full", "name": "OSWorld 2.0 (2026.06.24, full)", "group": "Agent 与工具", "unit": "%", "metric": "Binary task completion", "protocol": { "en": "Official end-to-end model runs, 500-step budget. Score is full task completion, not partial reward; reasoning settings are retained.", "zh": "官方端到端模型运行，最多 500 步。使用完整任务完成率，不使用部分奖励；保留推理设置。" }, "source": { "name": "OSWorld / XLANG Lab", "url": "https://osworld-v2.xlang.ai/", "data_url": "https://osworld-v2.xlang.ai/static/data/leaderboard/official-results.json?v=leaderboard-musespark-v10", "retrieved_at": "2026-11-01T16:45:15.576914+00:00", "sha256": "4cdcebeb33cd9c61cecfb275debc292326b48c209483d7da3ad66ee33d94b974", "selector": "v2026.06.24; full; 500 steps; official=true; modelType=e2e; binaryAccuracy / 100", "status": "dated_release", "kind": "official_leaderboard" }, "results": [ { "id": "3d3ca27333e4613abee0", "model": "Claude Opus 4.8", "organization": "Anthropic", "configuration": "max; batched tool; 500 steps", "score": 0.20600000000000002, "raw_score": 20.6, "raw_scale": 100, "source_record": "v2026.06.24|full|Claude Opus 4.8|max; batched tool; 500 steps" }, { "id": "909921c3eee71fdd77eb", "model": "Claude Opus 4.8", "organization": "Anthropic", "configuration": "max; standard; 500 steps", "score": 0.1852, "raw_score": 18.52, "raw_scale": 100, "source_record": "v2026.06.24|full|Claude Opus 4.8|max; standard; 500 steps" }, { "id": "4a3292fc9eb502df81c4", "model": "Claude Opus 4.7", "organization": "Anthropic", "configuration": "max; batched tool; 500 steps", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "v2026.06.24|full|Claude Opus 4.7|max; batched tool; 500 steps" }, { "id": "7cdc4d5e0211bd931cbb", "model": "Claude Opus 4.7", "organization": "Anthropic", "configuration": "max; standard; 500 steps", "score": 0.139, "raw_score": 13.9, "raw_scale": 100, "source_record": "v2026.06.24|full|Claude Opus 4.7|max; standard; 500 steps" }, { "id": "a30869717d6fffd666e2", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "xhigh; batch tool; 500 steps", "score": 0.13, "raw_score": 13.0, "raw_scale": 100, "source_record": "v2026.06.24|full|GPT-5.5|xhigh; batch tool; 500 steps" }, { "id": "28a321b9310814102cbf", "model": "Claude Sonnet 4.6", "organization": "Anthropic", "configuration": "medium; standard; 500 steps", "score": 0.09300000000000001, "raw_score": 9.3, "raw_scale": 100, "source_record": "v2026.06.24|full|Claude Sonnet 4.6|medium; standard; 500 steps" }, { "id": "3ddc97f95ae20c96b113", "model": "Claude Sonnet 4.6", "organization": "Anthropic", "configuration": "max; standard; 500 steps", "score": 0.083, "raw_score": 8.3, "raw_scale": 100, "source_record": "v2026.06.24|full|Claude Sonnet 4.6|max; standard; 500 steps" }, { "id": "fc686488a149807caff7", "model": "Kimi 2.6", "organization": "Moonshot AI", "configuration": "enabled; standard; 500 steps", "score": 0.046, "raw_score": 4.6, "raw_scale": 100, "source_record": "v2026.06.24|full|Kimi 2.6|enabled; standard; 500 steps" }, { "id": "39bfc661a604c789dc61", "model": "MiniMax M3", "organization": "MiniMax", "configuration": "enabled; standard; 500 steps", "score": 0.046, "raw_score": 4.6, "raw_scale": 100, "source_record": "v2026.06.24|full|MiniMax M3|enabled; standard; 500 steps" }, { "id": "0601ce4bf58df55d88bd", "model": "Qwen 3.7-Plus", "organization": "Alibaba", "configuration": "thinking; standard; 500 steps", "score": 0.027999999999999997, "raw_score": 2.8, "raw_scale": 100, "source_record": "v2026.06.24|full|Qwen 3.7-Plus|thinking; standard; 500 steps" } ], "public_history": true }, { "id": "monthly-swebench-2026-03", "name": "Monthly-SWEBench (2026-03)", "group": "编程", "unit": "%", "metric": "Pass@1", "source": { "data_url": "https://unipat.ai/benchmarks/MonthlySWEBench", "sha256": "62ff6c25c82a4ab4f1e3b77aedc559ceafb373dda864a6e4a675057c183a2894", "retrieved_at": "2026-11-02T05:58:00.809814+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/MonthlySWEBench", "selector": "SWE_DATA.data.Overall.snapshots[2026-03].passAt1, named batch and complete runs", "kind": "official_leaderboard", "status": "dated_release" }, "protocol": { "en": "2026-03 Overall, 112 tasks. The harness is recorded per result.", "zh": "2026-03 总榜共 112 道任务，各行保留执行框架。" }, "results": [ { "id": "73d96bfb6929bd6cebfe", "model": "Claude-Opus-4.6", "organization": "Anthropic", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "2026-03|Overall|1", "task_count": 112 }, { "id": "103797a0683e233bd2df", "model": "GLM-5.1", "organization": "Zhipu AI", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.6696, "raw_score": 66.96, "raw_scale": 100, "source_record": "2026-03|Overall|12", "task_count": 112 }, { "id": "7294b0f91d42e60db019", "model": "GPT-5.4-High", "organization": "OpenAI", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.643, "raw_score": 64.3, "raw_scale": 100, "source_record": "2026-03|Overall|2", "task_count": 112 }, { "id": "1b916c9d4a5624066af5", "model": "Qwen-3.6-Plus", "organization": "Alibaba", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.5714, "raw_score": 57.14, "raw_scale": 100, "source_record": "2026-03|Overall|10", "task_count": 112 }, { "id": "1d0bf94517af30af2a50", "model": "Gemini-3.1-Pro", "organization": "Google", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.5446, "raw_score": 54.46, "raw_scale": 100, "source_record": "2026-03|Overall|3", "task_count": 112 }, { "id": "f3a6a16baf2036bbc4f5", "model": "GLM-5", "organization": "Zhipu AI", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.5, "raw_score": 50.0, "raw_scale": 100, "source_record": "2026-03|Overall|5", "task_count": 112 }, { "id": "fc60ab8f74106fcda70c", "model": "Kimi-K2.5", "organization": "Moonshot AI", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.5, "raw_score": 50.0, "raw_scale": 100, "source_record": "2026-03|Overall|4", "task_count": 112 }, { "id": "0a3c5e6b8e965d131e2c", "model": "MiMo-V2-Pro", "organization": "Xiaomi", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.4643, "raw_score": 46.43, "raw_scale": 100, "source_record": "2026-03|Overall|11", "task_count": 112 }, { "id": "6af48027fba977aadb03", "model": "MiniMax-2.7", "organization": "MiniMax", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.4643, "raw_score": 46.43, "raw_scale": 100, "source_record": "2026-03|Overall|8", "task_count": 112 }, { "id": "78847911f134922cf5bd", "model": "Qwen-3.5-397B-A17B", "organization": "Alibaba", "configuration": "Terminus-2; 2026-03; 112 tasks", "score": 0.4375, "raw_score": 43.75, "raw_scale": 100, "source_record": "2026-03|Overall|6", "task_count": 112 } ], "public_history": true }, { "id": "monthly-swebench-2026-04", "name": "Monthly-SWEBench (2026-04)", "group": "编程", "unit": "%", "metric": "Pass@1", "source": { "data_url": "https://unipat.ai/benchmarks/MonthlySWEBench", "sha256": "62ff6c25c82a4ab4f1e3b77aedc559ceafb373dda864a6e4a675057c183a2894", "retrieved_at": "2026-11-02T05:58:00.809814+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/MonthlySWEBench", "selector": "SWE_DATA.data.Overall.snapshots[2026-04].passAt1, named batch and complete runs", "kind": "official_leaderboard", "status": "dated_release" }, "protocol": { "en": "2026-04 Overall, 90 tasks. The harness is recorded per result. Includes only runs covering all 90 tasks.", "zh": "2026-04 总榜共 90 道任务，各行保留执行框架。四月仅列跑完 90 题的记录。" }, "results": [ { "id": "4385b4ac0968834589e0", "model": "GPT-5.5-High", "organization": "OpenAI", "configuration": "OpenHands; 2026-04; 90 tasks", "score": 0.6444, "raw_score": 64.44, "raw_scale": 100, "source_record": "2026-04|Overall|2", "task_count": 90, "passed_tasks": 58 }, { "id": "44169cb1c536eb30948e", "model": "GLM-5.1-Thinking", "organization": "Zhipu AI", "configuration": "OpenHands; 2026-04; 90 tasks", "score": 0.6333, "raw_score": 63.33, "raw_scale": 100, "source_record": "2026-04|Overall|12", "task_count": 90, "passed_tasks": 57 }, { "id": "d5bb687fe611314c2d13", "model": "Claude-Opus-4.7-High", "organization": "Anthropic", "configuration": "OpenHands; 2026-04; 90 tasks", "score": 0.6222, "raw_score": 62.22, "raw_scale": 100, "source_record": "2026-04|Overall|1", "task_count": 90, "passed_tasks": 56 }, { "id": "a6ec8f955916d0ba7004", "model": "DeepSeek-V4-Pro-High", "organization": "DeepSeek", "configuration": "OpenHands; 2026-04; 90 tasks", "score": 0.6222, "raw_score": 62.22, "raw_scale": 100, "source_record": "2026-04|Overall|15", "task_count": 90, "passed_tasks": 56 }, { "id": "67e4ba9878bc670ef95f", "model": "Qwen-3.7-Max-Thinking", "organization": "Alibaba", "configuration": "OpenHands; 2026-04; 90 tasks", "score": 0.6111, "raw_score": 61.11, "raw_scale": 100, "source_record": "2026-04|Overall|17", "task_count": 90, "passed_tasks": 55 }, { "id": "67a24f0527d9babb5a45", "model": "Qwen-3.6-Plus-Thinking", "organization": "Alibaba", "configuration": "OpenHands; 2026-04; 90 tasks", "score": 0.5778, "raw_score": 57.78, "raw_scale": 100, "source_record": "2026-04|Overall|10", "task_count": 90, "passed_tasks": 52 }, { "id": "3b4406c46f9c2d8f94f6", "model": "Gemini-3.1-Pro-High", "organization": "Google", "configuration": "OpenHands; 2026-04; 90 tasks", "score": 0.5556, "raw_score": 55.56, "raw_scale": 100, "source_record": "2026-04|Overall|3", "task_count": 90, "passed_tasks": 50 }, { "id": "e55067b6d91bd122f6a8", "model": "Gemini-3.5-Flash-High", "organization": "Google", "configuration": "OpenHands; 2026-04; 90 tasks", "score": 0.5556, "raw_score": 55.56, "raw_scale": 100, "source_record": "2026-04|Overall|16", "task_count": 90, "passed_tasks": 50 } ], "public_history": true }, { "id": "monthly-swebench-2026-05", "name": "Monthly-SWEBench (2026-05)", "group": "编程", "unit": "%", "metric": "Pass@1", "source": { "data_url": "https://unipat.ai/benchmarks/MonthlySWEBench", "sha256": "62ff6c25c82a4ab4f1e3b77aedc559ceafb373dda864a6e4a675057c183a2894", "retrieved_at": "2026-11-02T05:58:00.809814+00:00", "name": "UniPat AI", "url": "https://unipat.ai/benchmarks/MonthlySWEBench", "selector": "SWE_DATA.data.Overall.snapshots[2026-05].passAt1, named batch and complete runs", "kind": "official_leaderboard", "status": "dated_release" }, "protocol": { "en": "2026-05 Overall, 100 tasks. The harness is recorded per result.", "zh": "2026-05 总榜共 100 道任务，各行保留执行框架。" }, "results": [ { "id": "68a5a9af7dcc6f273cfa", "model": "GPT-5.5-High", "organization": "OpenAI", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.82, "raw_score": 82.0, "raw_scale": 100, "source_record": "2026-05|Overall|13", "task_count": 100, "passed_tasks": 82, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "34d1b6f15bc2e39e94f7", "model": "Gemini-3.5-Flash-High", "organization": "Google", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.81, "raw_score": 81.0, "raw_scale": 100, "source_record": "2026-05|Overall|10", "task_count": 100, "passed_tasks": 81, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "1ab3814db23d092b3725", "model": "Claude-Opus-4.8-XHigh", "organization": "Anthropic", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.8, "raw_score": 80.0, "raw_scale": 100, "source_record": "2026-05|Overall|16", "task_count": 100, "passed_tasks": 80, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "cbfb68e5ca9656d6957f", "model": "Doubao-Seed-2.1-Pro-High", "organization": "ByteDance", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "2026-05|Overall|19", "task_count": 100, "passed_tasks": 75, "unfinished_count": 3, "unfinished_reasons": { "NonZeroAgentExitCodeError": 3 } }, { "id": "1b5003beec324cbbe913", "model": "GLM-5.2-Max", "organization": "Zhipu AI", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "2026-05|Overall|18", "task_count": 100, "passed_tasks": 75, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "9cd245419c18f647687f", "model": "Kimi-K2.7-Code-Thinking", "organization": "Moonshot AI", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.73, "raw_score": 73.0, "raw_scale": 100, "source_record": "2026-05|Overall|8", "task_count": 100, "passed_tasks": 73, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "6f0e3ad1a75d54a6e3b9", "model": "Qwen-3.7-Max-Thinking", "organization": "Alibaba", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "2026-05|Overall|12", "task_count": 100, "passed_tasks": 72, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "d3dadc3f96863cbe905f", "model": "GLM-5.1-Thinking", "organization": "Zhipu AI", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.71, "raw_score": 71.0, "raw_scale": 100, "source_record": "2026-05|Overall|14", "task_count": 100, "passed_tasks": 71, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "8d8307d77d4c6ed4a540", "model": "Hunyuan-HY3-Preview-High", "organization": "Tencent", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.71, "raw_score": 71.0, "raw_scale": 100, "source_record": "2026-05|Overall|17", "task_count": 100, "passed_tasks": 71, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "fa7ec2044d04abce53f5", "model": "DeepSeek-V4-Flash-High", "organization": "DeepSeek", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.69, "raw_score": 69.0, "raw_scale": 100, "source_record": "2026-05|Overall|6", "task_count": 100, "passed_tasks": 69, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "415ea434a2597472fe15", "model": "Kimi-K2.6-Thinking", "organization": "Moonshot AI", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.69, "raw_score": 69.0, "raw_scale": 100, "source_record": "2026-05|Overall|7", "task_count": 100, "passed_tasks": 69, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "4afe0faac91e26775b6d", "model": "Claude-Opus-4.7-High", "organization": "Anthropic", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.67, "raw_score": 67.0, "raw_scale": 100, "source_record": "2026-05|Overall|15", "task_count": 100, "passed_tasks": 67, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "a4244b2862c43a75733d", "model": "DeepSeek-V4-Pro-High", "organization": "DeepSeek", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.67, "raw_score": 67.0, "raw_scale": 100, "source_record": "2026-05|Overall|5", "task_count": 100, "passed_tasks": 67, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "a748d072d67e16eff63d", "model": "MiMo-V2.5-Pro-High", "organization": "Xiaomi", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.67, "raw_score": 67.0, "raw_scale": 100, "source_record": "2026-05|Overall|4", "task_count": 100, "passed_tasks": 67, "unfinished_count": 1, "unfinished_reasons": { "timeout:CancelledError": 1 } }, { "id": "ecfbb7c60c9b1b4bcd9e", "model": "MiniMax-M3-Adaptive", "organization": "MiniMax", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.67, "raw_score": 67.0, "raw_scale": 100, "source_record": "2026-05|Overall|2", "task_count": 100, "passed_tasks": 67, "unfinished_count": 1, "unfinished_reasons": { "timeout:CancelledError": 1 } }, { "id": "5681b4d3f1a2366e297a", "model": "Gemini-3.1-Pro-High", "organization": "Google", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.62, "raw_score": 62.0, "raw_scale": 100, "source_record": "2026-05|Overall|11", "task_count": 100, "passed_tasks": 62, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "0fed82c26638fe2525ad", "model": "Step-3.7-Flash-High", "organization": "StepFun", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.53, "raw_score": 53.0, "raw_scale": 100, "source_record": "2026-05|Overall|3", "task_count": 100, "passed_tasks": 53, "unfinished_count": 4, "unfinished_reasons": { "timeout:AgentTimeoutError": 4 } }, { "id": "4a0fba13f305a144e623", "model": "Doubao-Seed-2.0-Pro-High", "organization": "ByteDance", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.52, "raw_score": 52.0, "raw_scale": 100, "source_record": "2026-05|Overall|9", "task_count": 100, "passed_tasks": 52, "unfinished_count": 0, "unfinished_reasons": {} }, { "id": "26e2cead54ec995ee07d", "model": "MiniMax-M2.7-Reasoning", "organization": "MiniMax", "configuration": "OpenHands; 2026-05; 100 tasks", "score": 0.5, "raw_score": 50.0, "raw_scale": 100, "source_record": "2026-05|Overall|1", "task_count": 100, "passed_tasks": 50, "unfinished_count": 0, "unfinished_reasons": {} } ], "public_history": true }, { "id": "roadmapbench-terminus-resolved", "name": "RoadmapBench (Resolved, Terminus 2, 2026 paper)", "group": "编程", "unit": "%", "metric": "Resolved", "source": { "data_url": "https://arxiv.org/html/2605.15846v2", "sha256": "f8d46825b66278370841e87477dd02312e9e63d3f8b816be48c17a4c92ddc1c7", "retrieved_at": "2026-11-02T05:59:46.673715+00:00", "name": "UniPat AI", "url": "https://arxiv.org/html/2605.15846v2#S4.T2", "selector": "Table 2, Terminus 2 group, Resolved", "kind": "official_leaderboard", "status": "paper_baselines" }, "protocol": { "en": "Terminus 2 on 115 version-upgrade tasks, one trial per model.", "zh": "Terminus 2 执行 115 道版本升级任务，每个模型运行一次。" }, "results": [ { "id": "2e6cc37df07725802b3d", "model": "Claude-Opus-4.7", "organization": "Anthropic", "configuration": "Terminus 2; 115 tasks", "score": 0.38299999999999995, "raw_score": 38.3, "raw_scale": 100, "source_record": "Table 2|Claude-Opus-4.7|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.18" }, { "id": "8cf3ea6167ce10fc6483", "model": "Claude-Opus-4.6", "organization": "Anthropic", "configuration": "Terminus 2; 115 tasks", "score": 0.313, "raw_score": 31.3, "raw_scale": 100, "source_record": "Table 2|Claude-Opus-4.6|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.19" }, { "id": "5a51822dd299208c332d", "model": "GLM-5.1", "organization": "Z AI", "configuration": "Terminus 2; 115 tasks", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "Table 2|GLM-5.1|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.20" }, { "id": "ab6487da2cd16a21b66f", "model": "Qwen3.6-Plus", "organization": "Alibaba", "configuration": "Terminus 2; 115 tasks", "score": 0.165, "raw_score": 16.5, "raw_scale": 100, "source_record": "Table 2|Qwen3.6-Plus|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.21" }, { "id": "eed7ce285d75b5f8fab6", "model": "Kimi-K2.6", "organization": "Moonshot AI", "configuration": "Terminus 2; 115 tasks", "score": 0.157, "raw_score": 15.7, "raw_scale": 100, "source_record": "Table 2|Kimi-K2.6|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.22" }, { "id": "10304becefd3c22fe081", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "Terminus 2; 115 tasks", "score": 0.10400000000000001, "raw_score": 10.4, "raw_scale": 100, "source_record": "Table 2|DeepSeek-V4-Pro|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.23" }, { "id": "c82d3bcc11d88267d313", "model": "Mimo-V2.5-Pro", "organization": "Xiaomi", "configuration": "Terminus 2; 115 tasks", "score": 0.10400000000000001, "raw_score": 10.4, "raw_scale": 100, "source_record": "Table 2|Mimo-V2.5-Pro|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.24" }, { "id": "12f0a2ebdd5a9a725bab", "model": "Qwen3.5-397B", "organization": "Alibaba", "configuration": "Terminus 2; 115 tasks", "score": 0.10400000000000001, "raw_score": 10.4, "raw_scale": 100, "source_record": "Table 2|Qwen3.5-397B|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.25" }, { "id": "e8f46e6108877641a694", "model": "Kimi-K2.5", "organization": "Moonshot AI", "configuration": "Terminus 2; 115 tasks", "score": 0.078, "raw_score": 7.8, "raw_scale": 100, "source_record": "Table 2|Kimi-K2.5|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.26" }, { "id": "bd149ce7b5d8ec443b9d", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "Terminus 2; 115 tasks", "score": 0.043, "raw_score": 4.3, "raw_scale": 100, "source_record": "Table 2|MiniMax-M2.7|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.27" }, { "id": "57b596653ed9ef751535", "model": "Seed-2.0-Pro", "organization": "ByteDance", "configuration": "Terminus 2; 115 tasks", "score": 0.026000000000000002, "raw_score": 2.6, "raw_scale": 100, "source_record": "Table 2|Seed-2.0-Pro|Terminus 2|Resolved", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.28" } ], "public_history": true }, { "id": "roadmapbench-terminus-completion", "name": "RoadmapBench (Completion, Terminus 2, 2026 paper)", "group": "编程", "unit": "score", "metric": "Completion", "source": { "data_url": "https://arxiv.org/html/2605.15846v2", "sha256": "f8d46825b66278370841e87477dd02312e9e63d3f8b816be48c17a4c92ddc1c7", "retrieved_at": "2026-11-02T05:59:46.673715+00:00", "name": "UniPat AI", "url": "https://arxiv.org/html/2605.15846v2#S4.T2", "selector": "Table 2, Terminus 2 group, Completion", "kind": "official_leaderboard", "status": "paper_baselines" }, "protocol": { "en": "Terminus 2 on 115 version-upgrade tasks, one trial per model.", "zh": "Terminus 2 执行 115 道版本升级任务，每个模型运行一次。" }, "results": [ { "id": "398717358aac2094b240", "model": "Claude-Opus-4.7", "organization": "Anthropic", "configuration": "Terminus 2; 115 tasks", "score": 0.681, "raw_score": 0.681, "raw_scale": 1, "source_record": "Table 2|Claude-Opus-4.7|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.18" }, { "id": "806c2264c69f180e13d9", "model": "Claude-Opus-4.6", "organization": "Anthropic", "configuration": "Terminus 2; 115 tasks", "score": 0.666, "raw_score": 0.666, "raw_scale": 1, "source_record": "Table 2|Claude-Opus-4.6|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.19" }, { "id": "7bea560dd6e17db2ddf3", "model": "GLM-5.1", "organization": "Z AI", "configuration": "Terminus 2; 115 tasks", "score": 0.512, "raw_score": 0.512, "raw_scale": 1, "source_record": "Table 2|GLM-5.1|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.20" }, { "id": "2d9ef14114d675280cac", "model": "Qwen3.6-Plus", "organization": "Alibaba", "configuration": "Terminus 2; 115 tasks", "score": 0.508, "raw_score": 0.508, "raw_scale": 1, "source_record": "Table 2|Qwen3.6-Plus|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.21" }, { "id": "f52f66341751a2a5fdcf", "model": "Kimi-K2.6", "organization": "Moonshot AI", "configuration": "Terminus 2; 115 tasks", "score": 0.409, "raw_score": 0.409, "raw_scale": 1, "source_record": "Table 2|Kimi-K2.6|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.22" }, { "id": "b1c75223e619a83829df", "model": "DeepSeek-V4-Pro", "organization": "DeepSeek", "configuration": "Terminus 2; 115 tasks", "score": 0.395, "raw_score": 0.395, "raw_scale": 1, "source_record": "Table 2|DeepSeek-V4-Pro|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.23" }, { "id": "3d0827361de471900815", "model": "Kimi-K2.5", "organization": "Moonshot AI", "configuration": "Terminus 2; 115 tasks", "score": 0.36, "raw_score": 0.36, "raw_scale": 1, "source_record": "Table 2|Kimi-K2.5|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.26" }, { "id": "92ebf3ce836f5a321bd9", "model": "Mimo-V2.5-Pro", "organization": "Xiaomi", "configuration": "Terminus 2; 115 tasks", "score": 0.344, "raw_score": 0.344, "raw_scale": 1, "source_record": "Table 2|Mimo-V2.5-Pro|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.24" }, { "id": "3114a5a7d531ae88b51a", "model": "Qwen3.5-397B", "organization": "Alibaba", "configuration": "Terminus 2; 115 tasks", "score": 0.337, "raw_score": 0.337, "raw_scale": 1, "source_record": "Table 2|Qwen3.5-397B|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.25" }, { "id": "92ee739d0ccb0b439684", "model": "MiniMax-M2.7", "organization": "MiniMax", "configuration": "Terminus 2; 115 tasks", "score": 0.279, "raw_score": 0.279, "raw_scale": 1, "source_record": "Table 2|MiniMax-M2.7|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.27" }, { "id": "8d6d501baec01fb65e05", "model": "Seed-2.0-Pro", "organization": "ByteDance", "configuration": "Terminus 2; 115 tasks", "score": 0.135, "raw_score": 0.135, "raw_scale": 1, "source_record": "Table 2|Seed-2.0-Pro|Terminus 2|Completion", "task_count": 115, "report_url": "https://arxiv.org/html/2605.15846v2#S4.T2.8.1.28" } ], "public_history": true, "score_max": 1 }, { "id": "clawbench-v1-openclaw", "name": "ClawBench V1 (OpenClaw, 2026 paper)", "group": "Agent 与工具", "unit": "%", "metric": "Overall success rate", "source": { "data_url": "https://unipat.ai/benchmarks/ClawBench", "sha256": "83a2a04dd4649fe753ef60a1da641fba1eb174915421e805c301baf718a36a70", "retrieved_at": "2026-11-02T05:59:52.944069+00:00", "name": "ClawBench / UniPat AI", "url": "https://unipat.ai/benchmarks/ClawBench", "selector": "Eight-Model V1 Evaluation, Overall SR", "kind": "official_leaderboard", "status": "paper_baselines" }, "protocol": { "en": "Original OpenClaw evaluation on the 153-task V1 corpus.", "zh": "原版 OpenClaw 框架，V1 共 153 道任务。" }, "results": [ { "id": "2270c77f060f29cbd0f9", "model": "Claude Sonnet 4.6", "organization": "Anthropic", "configuration": "OpenClaw; V1; 153 tasks", "score": 0.33299999999999996, "raw_score": 33.3, "raw_scale": 100, "source_record": "Claude Sonnet 4.6|OpenClaw; V1; 153 tasks" }, { "id": "adb33f872fd930a75ed5", "model": "Qwen 3.5", "organization": "Alibaba", "configuration": "OpenClaw; V1; 153 tasks", "score": 0.261, "raw_score": 26.1, "raw_scale": 100, "source_record": "Qwen 3.5|OpenClaw; V1; 153 tasks" }, { "id": "0500728e90e9b652bd7f", "model": "GLM-5", "organization": "Z AI", "configuration": "OpenClaw; V1; 153 tasks", "score": 0.242, "raw_score": 24.2, "raw_scale": 100, "source_record": "GLM-5|OpenClaw; V1; 153 tasks" }, { "id": "be745db7db47bf66c728", "model": "Gemini 3 Flash", "organization": "Google", "configuration": "OpenClaw; V1; 153 tasks", "score": 0.19, "raw_score": 19.0, "raw_scale": 100, "source_record": "Gemini 3 Flash|OpenClaw; V1; 153 tasks" }, { "id": "b2f8b280bf89fe5af120", "model": "Claude Haiku 4.5", "organization": "Anthropic", "configuration": "OpenClaw; V1; 153 tasks", "score": 0.183, "raw_score": 18.3, "raw_scale": 100, "source_record": "Claude Haiku 4.5|OpenClaw; V1; 153 tasks" }, { "id": "877b726f2bc98f893c2e", "model": "Gemini 3.1 Pro", "organization": "Google", "configuration": "OpenClaw; V1; 153 tasks", "score": 0.098, "raw_score": 9.8, "raw_scale": 100, "source_record": "Gemini 3.1 Pro|OpenClaw; V1; 153 tasks" }, { "id": "be77f15df833f3ab81f8", "model": "GPT-5.4", "organization": "OpenAI", "configuration": "OpenClaw; V1; 153 tasks", "score": 0.065, "raw_score": 6.5, "raw_scale": 100, "source_record": "GPT-5.4|OpenClaw; V1; 153 tasks" }, { "id": "182788897b7cc48e84a2", "model": "Gemini 3.1 Flash Lite", "organization": "Google", "configuration": "OpenClaw; V1; 153 tasks", "score": 0.033, "raw_score": 3.3, "raw_scale": 100, "source_record": "Gemini 3.1 Flash Lite|OpenClaw; V1; 153 tasks" } ], "public_history": true }, { "id": "clawbench-v1-hermes", "name": "ClawBench V1 (Hermes, 2026-05-20)", "group": "Agent 与工具", "unit": "%", "metric": "Published V1 reward", "source": { "data_url": "https://claw-bench.com/api/leaderboard.json", "sha256": "7daa673365f612c0364b350e45f3f56f35f6d4dfd8f0aa5bcc986979c545a5e7", "retrieved_at": "2026-11-02T05:58:07.581706+00:00", "name": "ClawBench / UniPat AI", "url": "https://claw-bench.com/leaderboard", "selector": "rows_v1_hermes, full V1 rows, reward", "kind": "official_leaderboard", "status": "dated_release" }, "protocol": { "en": "Published V1 Hermes reward on 153 tasks.", "zh": "V1 题集共 153 道任务，使用发布方记录的 Hermes reward。" }, "results": [ { "id": "198e04e3271e6a55b4e5", "model": "claude-opus-4-6", "organization": "Anthropic", "configuration": "Hermes; V1; 153 tasks", "score": 0.6144, "raw_score": 0.6144, "raw_scale": 1, "source_record": "claude-opus-4-6|Hermes; V1; 153 tasks", "task_count": 153, "matched_tasks": 94 }, { "id": "5ebe02130bec868b5e49", "model": "claude-sonnet-4-6", "organization": "Anthropic", "configuration": "Hermes; V1; 153 tasks", "score": 0.5686, "raw_score": 0.5686, "raw_scale": 1, "source_record": "claude-sonnet-4-6|Hermes; V1; 153 tasks", "task_count": 153, "matched_tasks": 87 }, { "id": "8b64a3a12c02582fa597", "model": "claude-haiku-4-5-20251001", "organization": "Anthropic", "configuration": "Hermes; V1; 153 tasks", "score": 0.3007, "raw_score": 0.3007, "raw_scale": 1, "source_record": "claude-haiku-4-5-20251001|Hermes; V1; 153 tasks", "task_count": 153, "matched_tasks": 46 }, { "id": "1e7f7e88de376e76671f", "model": "gpt-5.4-2026-03-05", "organization": "OpenAI", "configuration": "Hermes; V1; 153 tasks", "score": 0.2549, "raw_score": 0.2549, "raw_scale": 1, "source_record": "gpt-5.4-2026-03-05|Hermes; V1; 153 tasks", "task_count": 153, "matched_tasks": 39 }, { "id": "f2a8a18b44d9006435a6", "model": "gpt-5.4-mini-2026-03-17", "organization": "OpenAI", "configuration": "Hermes; V1; 153 tasks", "score": 0.2484, "raw_score": 0.2484, "raw_scale": 1, "source_record": "gpt-5.4-mini-2026-03-17|Hermes; V1; 153 tasks", "task_count": 153, "matched_tasks": 38 }, { "id": "895b64e5b78a509d8ed1", "model": "kimi-k2.5", "organization": "Moonshot AI", "configuration": "Hermes; V1; 153 tasks", "score": 0.1765, "raw_score": 0.1765, "raw_scale": 1, "source_record": "kimi-k2.5|Hermes; V1; 153 tasks", "task_count": 153, "matched_tasks": 27 } ], "public_history": true }, { "id": "ceval-test-2023-0-shot-ao", "name": "C-Eval (2023 paper, 0-shot answer-only)", "group": "知识与推理", "unit": "%", "metric": "Mean subject accuracy", "protocol": { "en": "C-Eval paper, mean accuracy across 52 test subjects. 0-shot answer-only.", "zh": "C-Eval 原论文，取测试集的 52 门学科的平均准确率。0-shot AO。" }, "source": { "data_url": "https://arxiv.org/html/2305.08322v3", "sha256": "ae095c4515dad609755c223d0b685677916338a313d141b9283e26e501a30e42", "retrieved_at": "2026-11-02T06:00:07.109789+00:00", "name": "C-Eval authors", "url": "https://arxiv.org/html/2305.08322v3#S3.T3", "kind": "official_leaderboard", "status": "paper_baselines", "selector": "Paper v3, Table 3, Average; 52 test subjects; 0-shot answer-only; omit Random and missing cells" }, "results": [ { "id": "fd123610bf55ddcf46b1", "model": "GPT-4", "organization": "OpenAI", "configuration": "0-shot answer-only; 52 test subjects; API", "score": 0.664, "raw_score": 66.4, "raw_scale": 100, "source_record": "S3.T3|Average|GPT-4", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "4c6d65292e4941a162d0", "model": "ChatGPT", "organization": "OpenAI", "configuration": "0-shot answer-only; 52 test subjects; API", "score": 0.51, "raw_score": 51.0, "raw_scale": 100, "source_record": "S3.T3|Average|ChatGPT", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "be8f5a9f737813928b1c", "model": "Claude-v1.3", "organization": "Anthropic", "configuration": "0-shot answer-only; 52 test subjects; API", "score": 0.505, "raw_score": 50.5, "raw_scale": 100, "source_record": "S3.T3|Average|Claude-v1.3", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "ec3e4ce37596068e24fa", "model": "Bloomz-mt", "organization": "BigScience", "configuration": "0-shot answer-only; 52 test subjects; Weights", "score": 0.44299999999999995, "raw_score": 44.3, "raw_scale": 100, "source_record": "S3.T3|Average|Bloomz-mt", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "176B" }, { "id": "2f6d3359f5034e2dbaaa", "model": "GLM-130B", "organization": "Tsinghua", "configuration": "0-shot answer-only; 52 test subjects; Weights", "score": 0.44, "raw_score": 44.0, "raw_scale": 100, "source_record": "S3.T3|Average|GLM-130B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "130B" }, { "id": "276564f90e80037bd7b3", "model": "Claude-instant-v1.0", "organization": "Anthropic", "configuration": "0-shot answer-only; 52 test subjects; API", "score": 0.406, "raw_score": 40.6, "raw_scale": 100, "source_record": "S3.T3|Average|Claude-instant-v1.0", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "c092270b5227b46ba533", "model": "ChatGLM-6B", "organization": "Tsinghua", "configuration": "0-shot answer-only; 52 test subjects; Weights", "score": 0.389, "raw_score": 38.9, "raw_scale": 100, "source_record": "S3.T3|Average|ChatGLM-6B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "6B" }, { "id": "e7a78b2e6c6ea7edb67a", "model": "LLaMA-65B", "organization": "Meta", "configuration": "0-shot answer-only; 52 test subjects; Weights", "score": 0.34700000000000003, "raw_score": 34.7, "raw_scale": 100, "source_record": "S3.T3|Average|LLaMA-65B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "65B" }, { "id": "2e78cfe72cc94580b3be", "model": "MOSS", "organization": "Fudan", "configuration": "0-shot answer-only; 52 test subjects; Weights", "score": 0.331, "raw_score": 33.1, "raw_scale": 100, "source_record": "S3.T3|Average|MOSS", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "16B" }, { "id": "305a7256a6851d238369", "model": "Chinese-Alpaca-13B", "organization": "Cui et al.", "configuration": "0-shot answer-only; 52 test subjects; Weights", "score": 0.309, "raw_score": 30.9, "raw_scale": 100, "source_record": "S3.T3|Average|Chinese-Alpaca-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "13B" }, { "id": "10fb031378eb8eb3617f", "model": "Chinese-LLaMA-13B", "organization": "Cui et al.", "configuration": "0-shot answer-only; 52 test subjects; Weights", "score": 0.29600000000000004, "raw_score": 29.6, "raw_scale": 100, "source_record": "S3.T3|Average|Chinese-LLaMA-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T3", "source_date": "2023-11-06", "reported_parameters": "13B" } ], "public_history": true }, { "id": "ceval-test-2023-5-shot-ao", "name": "C-Eval (2023 paper, 5-shot answer-only)", "group": "知识与推理", "unit": "%", "metric": "Mean subject accuracy", "protocol": { "en": "C-Eval paper, mean accuracy across 52 test subjects. 5-shot answer-only.", "zh": "C-Eval 原论文，取测试集的 52 门学科的平均准确率。5-shot AO。" }, "source": { "data_url": "https://arxiv.org/html/2305.08322v3", "sha256": "ae095c4515dad609755c223d0b685677916338a313d141b9283e26e501a30e42", "retrieved_at": "2026-11-02T06:00:07.109789+00:00", "name": "C-Eval authors", "url": "https://arxiv.org/html/2305.08322v3#S3.T4", "kind": "official_leaderboard", "status": "paper_baselines", "selector": "Paper v3, Table 4, Average; 52 test subjects; 5-shot answer-only; omit Random and missing cells" }, "results": [ { "id": "0d3167eb146fb8f543a6", "model": "GPT-4", "organization": "OpenAI", "configuration": "5-shot answer-only; 52 test subjects; API", "score": 0.687, "raw_score": 68.7, "raw_scale": 100, "source_record": "S3.T4|Average|GPT-4", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "503c81a9018dd3e21449", "model": "ChatGPT", "organization": "OpenAI", "configuration": "5-shot answer-only; 52 test subjects; API", "score": 0.544, "raw_score": 54.4, "raw_scale": 100, "source_record": "S3.T4|Average|ChatGPT", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "d297faa476fa229835ed", "model": "Claude-v1.3", "organization": "Anthropic", "configuration": "5-shot answer-only; 52 test subjects; API", "score": 0.542, "raw_score": 54.2, "raw_scale": 100, "source_record": "S3.T4|Average|Claude-v1.3", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "f98c665217fe991afb05", "model": "Claude-instant-v1.0", "organization": "Anthropic", "configuration": "5-shot answer-only; 52 test subjects; API", "score": 0.45899999999999996, "raw_score": 45.9, "raw_scale": 100, "source_record": "S3.T4|Average|Claude-instant-v1.0", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "9ba83b294801fab5bb4f", "model": "GLM-130B", "organization": "Tsinghua", "configuration": "5-shot answer-only; 52 test subjects; Weights", "score": 0.40299999999999997, "raw_score": 40.3, "raw_scale": 100, "source_record": "S3.T4|Average|GLM-130B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "130B" }, { "id": "6787e0f391ffd6eea348", "model": "Bloomz-mt", "organization": "BigScience", "configuration": "5-shot answer-only; 52 test subjects; Weights", "score": 0.39, "raw_score": 39.0, "raw_scale": 100, "source_record": "S3.T4|Average|Bloomz-mt", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "176B" }, { "id": "f7a87c5f51ef5a852e5e", "model": "LLaMA-65B", "organization": "Meta", "configuration": "5-shot answer-only; 52 test subjects; Weights", "score": 0.38799999999999996, "raw_score": 38.8, "raw_scale": 100, "source_record": "S3.T4|Average|LLaMA-65B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "65B" }, { "id": "8572bfe13cb8766f1466", "model": "ChatGLM-6B", "organization": "Tsinghua", "configuration": "5-shot answer-only; 52 test subjects; Weights", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "S3.T4|Average|ChatGLM-6B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "6B" }, { "id": "d8d3650e6915bd700046", "model": "Chinese-LLaMA-13B", "organization": "Cui et al.", "configuration": "5-shot answer-only; 52 test subjects; Weights", "score": 0.33299999999999996, "raw_score": 33.3, "raw_scale": 100, "source_record": "S3.T4|Average|Chinese-LLaMA-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "13B" }, { "id": "13270f12c174d77f8b0e", "model": "MOSS", "organization": "Fudan", "configuration": "5-shot answer-only; 52 test subjects; Weights", "score": 0.311, "raw_score": 31.1, "raw_scale": 100, "source_record": "S3.T4|Average|MOSS", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "16B" }, { "id": "440efacdbbd6c1d1656e", "model": "Chinese-Alpaca-13B", "organization": "Cui et al.", "configuration": "5-shot answer-only; 52 test subjects; Weights", "score": 0.267, "raw_score": 26.7, "raw_scale": 100, "source_record": "S3.T4|Average|Chinese-Alpaca-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T4", "source_date": "2023-11-06", "reported_parameters": "13B" } ], "public_history": true }, { "id": "ceval-test-2023-5-shot-cot", "name": "C-Eval (2023 paper, 5-shot CoT)", "group": "知识与推理", "unit": "%", "metric": "Mean subject accuracy", "protocol": { "en": "C-Eval paper, mean accuracy across 52 test subjects. 5-shot CoT. The authors shortened the five-example prompt when it exceeded the context window.", "zh": "C-Eval 原论文，取测试集的 52 门学科的平均准确率。5-shot CoT。 五个示例超出模型上下文窗口时，作者会减少示例。" }, "source": { "data_url": "https://arxiv.org/html/2305.08322v3", "sha256": "ae095c4515dad609755c223d0b685677916338a313d141b9283e26e501a30e42", "retrieved_at": "2026-11-02T06:00:07.109789+00:00", "name": "C-Eval authors", "url": "https://arxiv.org/html/2305.08322v3#S3.T5", "kind": "official_leaderboard", "status": "paper_baselines", "selector": "Paper v3, Table 5, Average; 52 test subjects; 5-shot CoT; omit Random and missing cells" }, "results": [ { "id": "2cbc589389e6a28690b5", "model": "GPT-4", "organization": "OpenAI", "configuration": "5-shot CoT; 52 test subjects; API", "score": 0.6829999999999999, "raw_score": 68.3, "raw_scale": 100, "source_record": "S3.T5|Average|GPT-4", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T5", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "d4fc419639cf23d78032", "model": "Claude-v1.3", "organization": "Anthropic", "configuration": "5-shot CoT; 52 test subjects; API", "score": 0.542, "raw_score": 54.2, "raw_scale": 100, "source_record": "S3.T5|Average|Claude-v1.3", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T5", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "f06ea095bb03edf89338", "model": "ChatGPT", "organization": "OpenAI", "configuration": "5-shot CoT; 52 test subjects; API", "score": 0.5, "raw_score": 50.0, "raw_scale": 100, "source_record": "S3.T5|Average|ChatGPT", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T5", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "f3c8eb5750dbc34e8417", "model": "Claude-instant-v1.0", "organization": "Anthropic", "configuration": "5-shot CoT; 52 test subjects; API", "score": 0.445, "raw_score": 44.5, "raw_scale": 100, "source_record": "S3.T5|Average|Claude-instant-v1.0", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T5", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "44d3b99b01c2f13d09be", "model": "ChatGLM-6B", "organization": "Tsinghua", "configuration": "5-shot CoT; 52 test subjects; Weights", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "S3.T5|Average|ChatGLM-6B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T5", "source_date": "2023-11-06", "reported_parameters": "6B" }, { "id": "de859cc3049a1049824f", "model": "MOSS", "organization": "Fudan", "configuration": "5-shot CoT; 52 test subjects; Weights", "score": 0.312, "raw_score": 31.2, "raw_scale": 100, "source_record": "S3.T5|Average|MOSS", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T5", "source_date": "2023-11-06", "reported_parameters": "16B" }, { "id": "7cb140bfcd5fec626317", "model": "LLaMA-65B", "organization": "Meta", "configuration": "5-shot CoT; 52 test subjects; Weights", "score": 0.303, "raw_score": 30.3, "raw_scale": 100, "source_record": "S3.T5|Average|LLaMA-65B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T5", "source_date": "2023-11-06", "reported_parameters": "65B" }, { "id": "4ba68d763e5e884b1f12", "model": "GLM-130B", "organization": "Tsinghua", "configuration": "5-shot CoT; 52 test subjects; Weights", "score": 0.28800000000000003, "raw_score": 28.8, "raw_scale": 100, "source_record": "S3.T5|Average|GLM-130B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T5", "source_date": "2023-11-06", "reported_parameters": "130B" }, { "id": "bc302319bf26a8d7be7e", "model": "Chinese-LLaMA-13B", "organization": "Cui et al.", "configuration": "5-shot CoT; 52 test subjects; Weights", "score": 0.254, "raw_score": 25.4, "raw_scale": 100, "source_record": "S3.T5|Average|Chinese-LLaMA-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T5", "source_date": "2023-11-06", "reported_parameters": "13B" } ], "public_history": true }, { "id": "ceval-hard-2023-0-shot-ao", "name": "C-Eval Hard (2023 paper, 0-shot answer-only)", "group": "知识与推理", "unit": "%", "metric": "Mean subject accuracy", "protocol": { "en": "C-Eval paper, mean accuracy across eight hard test subjects. 0-shot answer-only.", "zh": "C-Eval 原论文，取测试集的 8 门高难学科的平均准确率。0-shot AO。" }, "source": { "data_url": "https://arxiv.org/html/2305.08322v3", "sha256": "ae095c4515dad609755c223d0b685677916338a313d141b9283e26e501a30e42", "retrieved_at": "2026-11-02T06:00:07.109789+00:00", "name": "C-Eval authors", "url": "https://arxiv.org/html/2305.08322v3#S3.T6", "kind": "official_leaderboard", "status": "paper_baselines", "selector": "Paper v3, Table 6, Zero-shot AO; eight hard test subjects; 0-shot answer-only; omit Random and missing cells" }, "results": [ { "id": "1931996cf2c568e4b454", "model": "GPT-4", "organization": "OpenAI", "configuration": "0-shot answer-only; eight hard test subjects; API", "score": 0.5329999999999999, "raw_score": 53.3, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|GPT-4", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "9a9131019450e31a6a48", "model": "Claude-v1.3", "organization": "Anthropic", "configuration": "0-shot answer-only; eight hard test subjects; API", "score": 0.376, "raw_score": 37.6, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|Claude-v1.3", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "75cb06f79e5a85476990", "model": "ChatGPT", "organization": "OpenAI", "configuration": "0-shot answer-only; eight hard test subjects; API", "score": 0.36700000000000005, "raw_score": 36.7, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|ChatGPT", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "626a54dc3a0da23bb9fc", "model": "Claude-instant-v1.0", "organization": "Anthropic", "configuration": "0-shot answer-only; eight hard test subjects; API", "score": 0.321, "raw_score": 32.1, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|Claude-instant-v1.0", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "47ec23a5ec5e030c1f8e", "model": "Bloomz-mt", "organization": "BigScience", "configuration": "0-shot answer-only; eight hard test subjects; Weights", "score": 0.308, "raw_score": 30.8, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|Bloomz-mt", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "176B" }, { "id": "6729bc00b96d17f30f26", "model": "GLM-130B", "organization": "Tsinghua", "configuration": "0-shot answer-only; eight hard test subjects; Weights", "score": 0.307, "raw_score": 30.7, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|GLM-130B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "130B" }, { "id": "d6f021b66837651dd6b9", "model": "LLaMA-65B", "organization": "Meta", "configuration": "0-shot answer-only; eight hard test subjects; Weights", "score": 0.298, "raw_score": 29.8, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|LLaMA-65B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "65B" }, { "id": "dab1f47fa72d23f31e14", "model": "ChatGLM-6B", "organization": "Tsinghua", "configuration": "0-shot answer-only; eight hard test subjects; Weights", "score": 0.292, "raw_score": 29.2, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|ChatGLM-6B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "6B" }, { "id": "3e441fe9852fb861f66f", "model": "MOSS", "organization": "Fudan", "configuration": "0-shot answer-only; eight hard test subjects; Weights", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|MOSS", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "16B" }, { "id": "0bea4c2a55bbe479905d", "model": "Chinese-LLaMA-13B", "organization": "Cui et al.", "configuration": "0-shot answer-only; eight hard test subjects; Weights", "score": 0.275, "raw_score": 27.5, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|Chinese-LLaMA-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "13B" }, { "id": "8b8530b4c52063aad93a", "model": "Chinese-Alpaca-13B", "organization": "Cui et al.", "configuration": "0-shot answer-only; eight hard test subjects; Weights", "score": 0.244, "raw_score": 24.4, "raw_scale": 100, "source_record": "S3.T6|Zero-shot AO|Chinese-Alpaca-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "13B" } ], "public_history": true }, { "id": "ceval-hard-2023-5-shot-ao", "name": "C-Eval Hard (2023 paper, 5-shot answer-only)", "group": "知识与推理", "unit": "%", "metric": "Mean subject accuracy", "protocol": { "en": "C-Eval paper, mean accuracy across eight hard test subjects. 5-shot answer-only.", "zh": "C-Eval 原论文，取测试集的 8 门高难学科的平均准确率。5-shot AO。" }, "source": { "data_url": "https://arxiv.org/html/2305.08322v3", "sha256": "ae095c4515dad609755c223d0b685677916338a313d141b9283e26e501a30e42", "retrieved_at": "2026-11-02T06:00:07.109789+00:00", "name": "C-Eval authors", "url": "https://arxiv.org/html/2305.08322v3#S3.T6", "kind": "official_leaderboard", "status": "paper_baselines", "selector": "Paper v3, Table 6, Five-shot AO; eight hard test subjects; 5-shot answer-only; omit Random and missing cells" }, "results": [ { "id": "450666c5e254634b0299", "model": "GPT-4", "organization": "OpenAI", "configuration": "5-shot answer-only; eight hard test subjects; API", "score": 0.5489999999999999, "raw_score": 54.9, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|GPT-4", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "ecdcefcca36c2ec8862a", "model": "ChatGPT", "organization": "OpenAI", "configuration": "5-shot answer-only; eight hard test subjects; API", "score": 0.414, "raw_score": 41.4, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|ChatGPT", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "7d366379ebabbee165cf", "model": "Claude-v1.3", "organization": "Anthropic", "configuration": "5-shot answer-only; eight hard test subjects; API", "score": 0.39, "raw_score": 39.0, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|Claude-v1.3", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "84e87270c07a66f68116", "model": "Claude-instant-v1.0", "organization": "Anthropic", "configuration": "5-shot answer-only; eight hard test subjects; API", "score": 0.355, "raw_score": 35.5, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|Claude-instant-v1.0", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "6f3bf6ff491d227dbcf4", "model": "LLaMA-65B", "organization": "Meta", "configuration": "5-shot answer-only; eight hard test subjects; Weights", "score": 0.317, "raw_score": 31.7, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|LLaMA-65B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "65B" }, { "id": "028aea224e841d47560c", "model": "Bloomz-mt", "organization": "BigScience", "configuration": "5-shot answer-only; eight hard test subjects; Weights", "score": 0.304, "raw_score": 30.4, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|Bloomz-mt", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "176B" }, { "id": "2f285bc81a4e09f797f0", "model": "GLM-130B", "organization": "Tsinghua", "configuration": "5-shot answer-only; eight hard test subjects; Weights", "score": 0.303, "raw_score": 30.3, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|GLM-130B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "130B" }, { "id": "734f8393a243d32dc242", "model": "Chinese-LLaMA-13B", "organization": "Cui et al.", "configuration": "5-shot answer-only; eight hard test subjects; Weights", "score": 0.273, "raw_score": 27.3, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|Chinese-LLaMA-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "13B" }, { "id": "ec4801eeff297d603382", "model": "Chinese-Alpaca-13B", "organization": "Cui et al.", "configuration": "5-shot answer-only; eight hard test subjects; Weights", "score": 0.271, "raw_score": 27.1, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|Chinese-Alpaca-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "13B" }, { "id": "b9180e3537bc3371f4f1", "model": "MOSS", "organization": "Fudan", "configuration": "5-shot answer-only; eight hard test subjects; Weights", "score": 0.24, "raw_score": 24.0, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|MOSS", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "16B" }, { "id": "bb06660b53dfedd3d888", "model": "ChatGLM-6B", "organization": "Tsinghua", "configuration": "5-shot answer-only; eight hard test subjects; Weights", "score": 0.231, "raw_score": 23.1, "raw_scale": 100, "source_record": "S3.T6|Five-shot AO|ChatGLM-6B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "6B" } ], "public_history": true }, { "id": "ceval-hard-2023-5-shot-cot", "name": "C-Eval Hard (2023 paper, 5-shot CoT)", "group": "知识与推理", "unit": "%", "metric": "Mean subject accuracy", "protocol": { "en": "C-Eval paper, mean accuracy across eight hard test subjects. 5-shot CoT. The authors shortened the five-example prompt when it exceeded the context window.", "zh": "C-Eval 原论文，取测试集的 8 门高难学科的平均准确率。5-shot CoT。 五个示例超出模型上下文窗口时，作者会减少示例。" }, "source": { "data_url": "https://arxiv.org/html/2305.08322v3", "sha256": "ae095c4515dad609755c223d0b685677916338a313d141b9283e26e501a30e42", "retrieved_at": "2026-11-02T06:00:07.109789+00:00", "name": "C-Eval authors", "url": "https://arxiv.org/html/2305.08322v3#S3.T6", "kind": "official_leaderboard", "status": "paper_baselines", "selector": "Paper v3, Table 6, Five-shot COT; eight hard test subjects; 5-shot CoT; omit Random and missing cells" }, "results": [ { "id": "8c208dd78a2ca92968d3", "model": "GPT-4", "organization": "OpenAI", "configuration": "5-shot CoT; eight hard test subjects; API", "score": 0.568, "raw_score": 56.8, "raw_scale": 100, "source_record": "S3.T6|Five-shot COT|GPT-4", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "9e3b35df46b669eee4c6", "model": "Claude-v1.3", "organization": "Anthropic", "configuration": "5-shot CoT; eight hard test subjects; API", "score": 0.392, "raw_score": 39.2, "raw_scale": 100, "source_record": "S3.T6|Five-shot COT|Claude-v1.3", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "a396055a8b1e7c1ac4a8", "model": "ChatGPT", "organization": "OpenAI", "configuration": "5-shot CoT; eight hard test subjects; API", "score": 0.35, "raw_score": 35.0, "raw_scale": 100, "source_record": "S3.T6|Five-shot COT|ChatGPT", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "edfaf5b060edf8de16fc", "model": "Claude-instant-v1.0", "organization": "Anthropic", "configuration": "5-shot CoT; eight hard test subjects; API", "score": 0.33399999999999996, "raw_score": 33.4, "raw_scale": 100, "source_record": "S3.T6|Five-shot COT|Claude-instant-v1.0", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "c12ad91c8aca654bcffb", "model": "ChatGLM-6B", "organization": "Tsinghua", "configuration": "5-shot CoT; eight hard test subjects; Weights", "score": 0.261, "raw_score": 26.1, "raw_scale": 100, "source_record": "S3.T6|Five-shot COT|ChatGLM-6B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "6B" }, { "id": "6d7096135126e3b6b81d", "model": "GLM-130B", "organization": "Tsinghua", "configuration": "5-shot CoT; eight hard test subjects; Weights", "score": 0.226, "raw_score": 22.6, "raw_scale": 100, "source_record": "S3.T6|Five-shot COT|GLM-130B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "130B" }, { "id": "7d3ca9f0bcbfe5c67fc2", "model": "MOSS", "organization": "Fudan", "configuration": "5-shot CoT; eight hard test subjects; Weights", "score": 0.21600000000000003, "raw_score": 21.6, "raw_scale": 100, "source_record": "S3.T6|Five-shot COT|MOSS", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "16B" }, { "id": "09d9d673b18ee859c922", "model": "LLaMA-65B", "organization": "Meta", "configuration": "5-shot CoT; eight hard test subjects; Weights", "score": 0.214, "raw_score": 21.4, "raw_scale": 100, "source_record": "S3.T6|Five-shot COT|LLaMA-65B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "65B" }, { "id": "5a4224434086027a994a", "model": "Chinese-LLaMA-13B", "organization": "Cui et al.", "configuration": "5-shot CoT; eight hard test subjects; Weights", "score": 0.154, "raw_score": 15.4, "raw_scale": 100, "source_record": "S3.T6|Five-shot COT|Chinese-LLaMA-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T6", "source_date": "2023-11-06", "reported_parameters": "13B" } ], "public_history": true }, { "id": "ceval-validation-2023-0-shot-ao", "name": "C-Eval (2023 paper, validation, 0-shot answer-only)", "group": "知识与推理", "unit": "%", "metric": "Mean subject accuracy", "protocol": { "en": "C-Eval paper, mean accuracy across 52 validation subjects. 0-shot answer-only.", "zh": "C-Eval 原论文，取验证集的 52 门学科的平均准确率。0-shot AO。" }, "source": { "data_url": "https://arxiv.org/html/2305.08322v3", "sha256": "ae095c4515dad609755c223d0b685677916338a313d141b9283e26e501a30e42", "retrieved_at": "2026-11-02T06:00:07.109789+00:00", "name": "C-Eval authors", "url": "https://arxiv.org/html/2305.08322v3#S3.T7", "kind": "official_leaderboard", "status": "paper_baselines", "selector": "Paper v3, Table 7, Zero-shot; 52 validation subjects; 0-shot answer-only; omit Random and missing cells" }, "results": [ { "id": "e3a2f219fc93c585cf3e", "model": "GPT-4", "organization": "OpenAI", "configuration": "0-shot answer-only; 52 validation subjects; API", "score": 0.667, "raw_score": 66.7, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|GPT-4", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "991cfec4989d2cf04ca4", "model": "Claude-v1.3", "organization": "Anthropic", "configuration": "0-shot answer-only; 52 validation subjects; API", "score": 0.521, "raw_score": 52.1, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|Claude-v1.3", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "e65f39affc2a153c70c4", "model": "ChatGPT", "organization": "OpenAI", "configuration": "0-shot answer-only; 52 validation subjects; API", "score": 0.508, "raw_score": 50.8, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|ChatGPT", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "bd6e7afa178ecdda015e", "model": "Bloomz-mt", "organization": "BigScience", "configuration": "0-shot answer-only; 52 validation subjects; Weights", "score": 0.45899999999999996, "raw_score": 45.9, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|Bloomz-mt", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "176B" }, { "id": "38fb2fc7d5fa428a73ce", "model": "GLM-130B", "organization": "Tsinghua", "configuration": "0-shot answer-only; 52 validation subjects; Weights", "score": 0.442, "raw_score": 44.2, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|GLM-130B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "130B" }, { "id": "ef27974c3b5c039080c6", "model": "Claude-instant-v1.0", "organization": "Anthropic", "configuration": "0-shot answer-only; 52 validation subjects; API", "score": 0.43200000000000005, "raw_score": 43.2, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|Claude-instant-v1.0", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "967b0d57aa63b64f8e01", "model": "ChatGLM-6B", "organization": "Tsinghua", "configuration": "0-shot answer-only; 52 validation subjects; Weights", "score": 0.397, "raw_score": 39.7, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|ChatGLM-6B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "6B" }, { "id": "57d74f82ed43a6485caa", "model": "LLaMA-65B", "organization": "Meta", "configuration": "0-shot answer-only; 52 validation subjects; Weights", "score": 0.386, "raw_score": 38.6, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|LLaMA-65B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "65B" }, { "id": "779750a1d4d9d3f3311a", "model": "MOSS", "organization": "Fudan", "configuration": "0-shot answer-only; 52 validation subjects; Weights", "score": 0.35100000000000003, "raw_score": 35.1, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|MOSS", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "16B" }, { "id": "8fc18aa684b8ab6f893f", "model": "Chinese-Alpaca-13B", "organization": "Cui et al.", "configuration": "0-shot answer-only; 52 validation subjects; Weights", "score": 0.32, "raw_score": 32.0, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|Chinese-Alpaca-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "13B" }, { "id": "7caa99d8da20bfbe6982", "model": "Chinese-LLaMA-13B", "organization": "Cui et al.", "configuration": "0-shot answer-only; 52 validation subjects; Weights", "score": 0.294, "raw_score": 29.4, "raw_scale": 100, "source_record": "S3.T7|Zero-shot|Chinese-LLaMA-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "13B" } ], "public_history": true }, { "id": "ceval-validation-2023-5-shot-ao", "name": "C-Eval (2023 paper, validation, 5-shot answer-only)", "group": "知识与推理", "unit": "%", "metric": "Mean subject accuracy", "protocol": { "en": "C-Eval paper, mean accuracy across 52 validation subjects. 5-shot answer-only.", "zh": "C-Eval 原论文，取验证集的 52 门学科的平均准确率。5-shot AO。" }, "source": { "data_url": "https://arxiv.org/html/2305.08322v3", "sha256": "ae095c4515dad609755c223d0b685677916338a313d141b9283e26e501a30e42", "retrieved_at": "2026-11-02T06:00:07.109789+00:00", "name": "C-Eval authors", "url": "https://arxiv.org/html/2305.08322v3#S3.T7", "kind": "official_leaderboard", "status": "paper_baselines", "selector": "Paper v3, Table 7, Five-shot; 52 validation subjects; 5-shot answer-only; omit Random and missing cells" }, "results": [ { "id": "a6c8c0ecd12e59c3a131", "model": "GPT-4", "organization": "OpenAI", "configuration": "5-shot answer-only; 52 validation subjects; API", "score": 0.6990000000000001, "raw_score": 69.9, "raw_scale": 100, "source_record": "S3.T7|Five-shot|GPT-4", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "8275241acbbeeb2418b8", "model": "Claude-v1.3", "organization": "Anthropic", "configuration": "5-shot answer-only; 52 validation subjects; API", "score": 0.555, "raw_score": 55.5, "raw_scale": 100, "source_record": "S3.T7|Five-shot|Claude-v1.3", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "54715df5add37385820f", "model": "ChatGPT", "organization": "OpenAI", "configuration": "5-shot answer-only; 52 validation subjects; API", "score": 0.535, "raw_score": 53.5, "raw_scale": 100, "source_record": "S3.T7|Five-shot|ChatGPT", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "5da416183f3eef06b793", "model": "Claude-instant-v1.0", "organization": "Anthropic", "configuration": "5-shot answer-only; 52 validation subjects; API", "score": 0.474, "raw_score": 47.4, "raw_scale": 100, "source_record": "S3.T7|Five-shot|Claude-instant-v1.0", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "undisclosed" }, { "id": "3503366fb0704c268abb", "model": "GLM-130B", "organization": "Tsinghua", "configuration": "5-shot answer-only; 52 validation subjects; Weights", "score": 0.408, "raw_score": 40.8, "raw_scale": 100, "source_record": "S3.T7|Five-shot|GLM-130B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "130B" }, { "id": "e53e9defc6d05429f1b0", "model": "LLaMA-65B", "organization": "Meta", "configuration": "5-shot answer-only; 52 validation subjects; Weights", "score": 0.39799999999999996, "raw_score": 39.8, "raw_scale": 100, "source_record": "S3.T7|Five-shot|LLaMA-65B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "65B" }, { "id": "ef8faef2336c5f63117f", "model": "Bloomz-mt", "organization": "BigScience", "configuration": "5-shot answer-only; 52 validation subjects; Weights", "score": 0.38, "raw_score": 38.0, "raw_scale": 100, "source_record": "S3.T7|Five-shot|Bloomz-mt", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "176B" }, { "id": "0463993e1d94521c48b5", "model": "ChatGLM-6B", "organization": "Tsinghua", "configuration": "5-shot answer-only; 52 validation subjects; Weights", "score": 0.371, "raw_score": 37.1, "raw_scale": 100, "source_record": "S3.T7|Five-shot|ChatGLM-6B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "6B" }, { "id": "0b0641ffad1352aba4b0", "model": "Chinese-LLaMA-13B", "organization": "Cui et al.", "configuration": "5-shot answer-only; 52 validation subjects; Weights", "score": 0.331, "raw_score": 33.1, "raw_scale": 100, "source_record": "S3.T7|Five-shot|Chinese-LLaMA-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "13B" }, { "id": "7df2298cf60352337879", "model": "MOSS", "organization": "Fudan", "configuration": "5-shot answer-only; 52 validation subjects; Weights", "score": 0.289, "raw_score": 28.9, "raw_scale": 100, "source_record": "S3.T7|Five-shot|MOSS", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "16B" }, { "id": "cf203fffa6ddeb9c0df2", "model": "Chinese-Alpaca-13B", "organization": "Cui et al.", "configuration": "5-shot answer-only; 52 validation subjects; Weights", "score": 0.272, "raw_score": 27.2, "raw_scale": 100, "source_record": "S3.T7|Five-shot|Chinese-Alpaca-13B", "report_url": "https://arxiv.org/html/2305.08322v3#S3.T7", "source_date": "2023-11-06", "reported_parameters": "13B" } ], "public_history": true }, { "id": "gsm8k-wei-2022-standard-1-order", "name": "GSM8K (Wei et al. 2022, 8-shot standard, one exemplar order)", "group": "数学与科学", "unit": "%", "metric": "Test accuracy", "protocol": { "en": "1,319 GSM8K test questions, eight examples and greedy decoding. One exemplar order.", "zh": "GSM8K 测试集 1,319 题，使用八个示例和贪心解码。使用一种示例顺序。" }, "source": { "data_url": "https://arxiv.org/html/2201.11903v6", "sha256": "018cd3d1b283350c419b2c54ecc4c0a81351f2b78dea77ae0b6fdf49e47242c1", "retrieved_at": "2026-11-02T05:58:58.152248+00:00", "name": "Wei et al. / Google Research", "url": "https://arxiv.org/html/2201.11903v6#A2.T2", "kind": "evaluator", "status": "dated_release", "selector": "Wei et al. paper v6, Table 2, GSM8K / standard; one exemplar order; see Section 3.1" }, "results": [ { "id": "714958266e311ac184d7", "model": "code-davinci-002", "organization": "OpenAI", "configuration": "8-shot standard; greedy decoding; 1 exemplar order", "score": 0.19699999999999998, "raw_score": 19.7, "raw_scale": 100, "source_record": "A2.T2|GSM8K|Codex|-|standard", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "Codex" }, { "id": "42d74a49841638b30d00", "model": "PaLM 540B", "organization": "Google", "configuration": "8-shot standard; greedy decoding; 1 exemplar order", "score": 0.179, "raw_score": 17.9, "raw_scale": 100, "source_record": "A2.T2|GSM8K|PaLM|540B|standard", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "PaLM 540B" }, { "id": "6901c53ef25277986e7c", "model": "text-davinci-002", "organization": "OpenAI", "configuration": "8-shot standard; greedy decoding; 1 exemplar order", "score": 0.156, "raw_score": 15.6, "raw_scale": 100, "source_record": "A2.T2|GSM8K|GPT|175B|standard", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "GPT 175B" }, { "id": "7d7bc49300fff5286b72", "model": "PaLM 62B", "organization": "Google", "configuration": "8-shot standard; greedy decoding; 1 exemplar order", "score": 0.096, "raw_score": 9.6, "raw_scale": 100, "source_record": "A2.T2|GSM8K|PaLM|62B|standard", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "PaLM 62B" }, { "id": "57b213864123aead6852", "model": "PaLM 8B", "organization": "Google", "configuration": "8-shot standard; greedy decoding; 1 exemplar order", "score": 0.049, "raw_score": 4.9, "raw_scale": 100, "source_record": "A2.T2|GSM8K|PaLM|8B|standard", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "PaLM 8B" }, { "id": "e5ec920807b7fdc476c6", "model": "UL2 20B", "organization": "Google", "configuration": "8-shot standard; greedy decoding; 1 exemplar order", "score": 0.040999999999999995, "raw_score": 4.1, "raw_scale": 100, "source_record": "A2.T2|GSM8K|UL2|20B|standard", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "UL2 20B" }, { "id": "d2a15dc5a732f7e52a52", "model": "text-curie-001", "organization": "OpenAI", "configuration": "8-shot standard; greedy decoding; 1 exemplar order", "score": 0.04, "raw_score": 4.0, "raw_scale": 100, "source_record": "A2.T2|GSM8K|GPT|6.7B|standard", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "GPT 6.7B" }, { "id": "272698b6154e1a188839", "model": "text-babbage-001", "organization": "OpenAI", "configuration": "8-shot standard; greedy decoding; 1 exemplar order", "score": 0.024, "raw_score": 2.4, "raw_scale": 100, "source_record": "A2.T2|GSM8K|GPT|1.3B|standard", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "GPT 1.3B" }, { "id": "5aab0fc9ac61923df4d4", "model": "text-ada-001", "organization": "OpenAI", "configuration": "8-shot standard; greedy decoding; 1 exemplar order", "score": 0.022000000000000002, "raw_score": 2.2, "raw_scale": 100, "source_record": "A2.T2|GSM8K|GPT|350M|standard", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "GPT 350M" } ], "public_history": true }, { "id": "gsm8k-wei-2022-cot-1-order", "name": "GSM8K (Wei et al. 2022, 8-shot CoT, one exemplar order)", "group": "数学与科学", "unit": "%", "metric": "Test accuracy", "protocol": { "en": "1,319 GSM8K test questions, eight examples and greedy decoding. One exemplar order.", "zh": "GSM8K 测试集 1,319 题，使用八个示例和贪心解码。使用一种示例顺序。" }, "source": { "data_url": "https://arxiv.org/html/2201.11903v6", "sha256": "018cd3d1b283350c419b2c54ecc4c0a81351f2b78dea77ae0b6fdf49e47242c1", "retrieved_at": "2026-11-02T05:58:58.152248+00:00", "name": "Wei et al. / Google Research", "url": "https://arxiv.org/html/2201.11903v6#A2.T2", "kind": "evaluator", "status": "dated_release", "selector": "Wei et al. paper v6, Table 2, GSM8K / CoT; one exemplar order; see Section 3.1" }, "results": [ { "id": "ea261b19cac18c2c3476", "model": "code-davinci-002", "organization": "OpenAI", "configuration": "8-shot CoT; greedy decoding; 1 exemplar order", "score": 0.631, "raw_score": 63.1, "raw_scale": 100, "source_record": "A2.T2|GSM8K|Codex|-|CoT", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "Codex" }, { "id": "d82dd6c24f144405dc0f", "model": "PaLM 540B", "organization": "Google", "configuration": "8-shot CoT; greedy decoding; 1 exemplar order", "score": 0.569, "raw_score": 56.9, "raw_scale": 100, "source_record": "A2.T2|GSM8K|PaLM|540B|CoT", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "PaLM 540B" }, { "id": "9bbf657799d4ad274568", "model": "text-davinci-002", "organization": "OpenAI", "configuration": "8-shot CoT; greedy decoding; 1 exemplar order", "score": 0.469, "raw_score": 46.9, "raw_scale": 100, "source_record": "A2.T2|GSM8K|GPT|175B|CoT", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "GPT 175B" }, { "id": "0fe8b96bf6ba14864b04", "model": "PaLM 62B", "organization": "Google", "configuration": "8-shot CoT; greedy decoding; 1 exemplar order", "score": 0.299, "raw_score": 29.9, "raw_scale": 100, "source_record": "A2.T2|GSM8K|PaLM|62B|CoT", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "PaLM 62B" }, { "id": "9cc9fafc431576358399", "model": "UL2 20B", "organization": "Google", "configuration": "8-shot CoT; greedy decoding; 1 exemplar order", "score": 0.044000000000000004, "raw_score": 4.4, "raw_scale": 100, "source_record": "A2.T2|GSM8K|UL2|20B|CoT", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "UL2 20B" }, { "id": "68142b7e53df0101b81c", "model": "PaLM 8B", "organization": "Google", "configuration": "8-shot CoT; greedy decoding; 1 exemplar order", "score": 0.040999999999999995, "raw_score": 4.1, "raw_scale": 100, "source_record": "A2.T2|GSM8K|PaLM|8B|CoT", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "PaLM 8B" }, { "id": "25decf014c33f847c2f6", "model": "text-curie-001", "organization": "OpenAI", "configuration": "8-shot CoT; greedy decoding; 1 exemplar order", "score": 0.024, "raw_score": 2.4, "raw_scale": 100, "source_record": "A2.T2|GSM8K|GPT|6.7B|CoT", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "GPT 6.7B" }, { "id": "01e326db75f00c398b81", "model": "text-ada-001", "organization": "OpenAI", "configuration": "8-shot CoT; greedy decoding; 1 exemplar order", "score": 0.005, "raw_score": 0.5, "raw_scale": 100, "source_record": "A2.T2|GSM8K|GPT|350M|CoT", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "GPT 350M" }, { "id": "a36835856c7fae4faa2a", "model": "text-babbage-001", "organization": "OpenAI", "configuration": "8-shot CoT; greedy decoding; 1 exemplar order", "score": 0.005, "raw_score": 0.5, "raw_scale": 100, "source_record": "A2.T2|GSM8K|GPT|1.3B|CoT", "report_url": "https://arxiv.org/html/2201.11903v6#A2.T2", "source_date": "2023-01-10", "source_model": "GPT 1.3B" } ], "public_history": true }, { "id": "mmlu-helm", "name": "MMLU (HELM 2025, v1.13.0, 5-shot)", "group": "知识与推理", "unit": "%", "metric": "Mean subject exact match", "protocol": { "en": "HELM v1.13.0 (2025-01-10), five-shot. Mean exact match across all 57 subjects, 14,042 test instances.", "zh": "HELM v1.13.0（2025-01-10），五样本。共 14,042 道测试题，取 57 门学科精确匹配率的平均值。" }, "source": { "data_url": "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu.json", "sha256": "18ad1a09b793ce16c7dc98d805b63feef411e64f74ba5483b115eecde716d271", "retrieved_at": "2026-11-02T05:58:58.299597+00:00", "name": "Stanford CRFM / HELM", "url": "https://crfm.stanford.edu/helm/mmlu/latest/", "kind": "evaluator", "status": "dated_release", "selector": "HELM v1.13.0; Massive Multitask Language Understanding (MMLU) All Subjects; # train=5; EM; 14042 instances per model", "artifacts": [ { "data_url": "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/summary.json", "sha256": "d4eb487e79bcdd7329bdcb10f5116364c2501842fe5e58da6e0bd717f90ee71e", "retrieved_at": "2026-11-01T16:51:20.444054+00:00" } ] }, "results": [ { "id": "006d28473f9cfd368ef3", "model": "Claude 3.5 Sonnet (20241022)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8727304053073446, "raw_score": 0.8727304053073446, "raw_scale": 1, "source_record": "Claude 3.5 Sonnet (20241022)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20241022,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "54381ac6ba0eb80c714f", "model": "DeepSeek v3", "organization": "DeepSeek", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8720324162906944, "raw_score": 0.8720324162906944, "raw_scale": 1, "source_record": "DeepSeek v3|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=deepseek-ai_deepseek-v3,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "38687459163d5e08b8a1", "model": "Gemini 1.5 Pro (002)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8687880474746731, "raw_score": 0.8687880474746731, "raw_scale": 1, "source_record": "Gemini 1.5 Pro (002)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemini-1.5-pro-002,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "98f99e020e5fbd3630b2", "model": "Claude 3.5 Sonnet (20240620)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8654078253932308, "raw_score": 0.8654078253932308, "raw_scale": 1, "source_record": "Claude 3.5 Sonnet (20240620)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=anthropic_claude-3-5-sonnet-20240620,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "cf286a1a014709eb6711", "model": "Claude 3 Opus (20240229)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8457152123680285, "raw_score": 0.8457152123680285, "raw_scale": 1, "source_record": "Claude 3 Opus (20240229)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=anthropic_claude-3-opus-20240229,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "6206c7fa83ce5a8bf096", "model": "Llama 3.1 Instruct Turbo (405B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8453174958174847, "raw_score": 0.8453174958174847, "raw_scale": 1, "source_record": "Llama 3.1 Instruct Turbo (405B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-3.1-405b-instruct-turbo,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "7ea442961d7024462f72", "model": "GPT-4o (2024-08-06)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8433429854407875, "raw_score": 0.8433429854407875, "raw_scale": 1, "source_record": "GPT-4o (2024-08-06)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=openai_gpt-4o-2024-08-06,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "feef006eada04c785a18", "model": "GPT-4o (2024-05-13)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8417289823699189, "raw_score": 0.8417289823699189, "raw_scale": 1, "source_record": "GPT-4o (2024-05-13)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=openai_gpt-4o-2024-05-13,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "67627e618af8792c0903", "model": "Qwen2.5 Instruct Turbo (72B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8337220858502318, "raw_score": 0.8337220858502318, "raw_scale": 1, "source_record": "Qwen2.5 Instruct Turbo (72B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=qwen_qwen2.5-72b-instruct-turbo,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "c2b113ce7f12a2fdc8b0", "model": "Gemini 1.5 Pro (001)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8270842647299882, "raw_score": 0.8270842647299882, "raw_scale": 1, "source_record": "Gemini 1.5 Pro (001)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemini-1.5-pro-001,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "8eba709563aa5d55adfc", "model": "GPT-4 (0613)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8243666982319177, "raw_score": 0.8243666982319177, "raw_scale": 1, "source_record": "GPT-4 (0613)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=openai_gpt-4-0613,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "771890a08222faef0a88", "model": "Qwen2 Instruct (72B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8235031827495974, "raw_score": 0.8235031827495974, "raw_scale": 1, "source_record": "Qwen2 Instruct (72B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=qwen_qwen2-72b-instruct,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "abeb75ee822ba470ebc2", "model": "Amazon Nova Pro", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8199706336653506, "raw_score": 0.8199706336653506, "raw_scale": 1, "source_record": "Amazon Nova Pro|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=amazon_nova-pro-v1:0,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "03ae695d911db561e34d", "model": "Palmyra-X-004", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8127981488830059, "raw_score": 0.8127981488830059, "raw_scale": 1, "source_record": "Palmyra-X-004|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=writer_palmyra-x-004,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "3cdeb06fa5ee896dfcde", "model": "GPT-4 Turbo (2024-04-09)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8127187978959117, "raw_score": 0.8127187978959117, "raw_scale": 1, "source_record": "GPT-4 Turbo (2024-04-09)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=openai_gpt-4-turbo-2024-04-09,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "5f444588240ec4b35392", "model": "Gemini 1.5 Pro (0409 preview)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8095085603987299, "raw_score": 0.8095085603987299, "raw_scale": 1, "source_record": "Gemini 1.5 Pro (0409 preview)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemini-1.5-pro-preview-0409,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "2ba4613fd2430d8929c9", "model": "Llama 3.2 Vision Instruct Turbo (90B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8026595003973722, "raw_score": 0.8026595003973722, "raw_scale": 1, "source_record": "Llama 3.2 Vision Instruct Turbo (90B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-3.2-90b-vision-instruct-turbo,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "7205338d35bd065f5bc4", "model": "Llama 3.1 Instruct Turbo (70B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.8010355128348475, "raw_score": 0.8010355128348475, "raw_scale": 1, "source_record": "Llama 3.1 Instruct Turbo (70B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-3.1-70b-instruct-turbo,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "40e1bc890785a56d27b2", "model": "Mistral Large 2 (2407)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7999764497205896, "raw_score": 0.7999764497205896, "raw_scale": 1, "source_record": "Mistral Large 2 (2407)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=mistralai_mistral-large-2407,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "9b7828798a186365327b", "model": "Gemini 2.0 Flash (Experimental)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7973847335945918, "raw_score": 0.7973847335945918, "raw_scale": 1, "source_record": "Gemini 2.0 Flash (Experimental)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemini-2.0-flash-exp,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "492b8abca0ae72686ae0", "model": "GPT-4 Turbo (1106 preview)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7955114613915927, "raw_score": 0.7955114613915927, "raw_scale": 1, "source_record": "GPT-4 Turbo (1106 preview)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=openai_gpt-4-1106-preview,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "7a75478e8f9b2562a6c7", "model": "Llama 3 (70B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7933654187607483, "raw_score": 0.7933654187607483, "raw_scale": 1, "source_record": "Llama 3 (70B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-3-70b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "1fae4cadaa77e68b47b6", "model": "Yi Large (Preview)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7932309410233613, "raw_score": 0.7932309410233613, "raw_scale": 1, "source_record": "Yi Large (Preview)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=01-ai_yi-large-preview,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "0932650711d62d8eb9f8", "model": "Llama 3.3 Instruct Turbo (70B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.790602761302675, "raw_score": 0.790602761302675, "raw_scale": 1, "source_record": "Llama 3.3 Instruct Turbo (70B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-3.3-70b-instruct-turbo,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "2570c4d573a4c3a0e46d", "model": "Palmyra X V3 (72B)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7861802691450123, "raw_score": 0.7861802691450123, "raw_scale": 1, "source_record": "Palmyra X V3 (72B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=writer_palmyra-x-v3,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "3e48718d1484e47d835b", "model": "PaLM-2 (Unicorn)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.78602059579238, "raw_score": 0.78602059579238, "raw_scale": 1, "source_record": "PaLM-2 (Unicorn)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_text-unicorn@001,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "10de1cf45b29098b316b", "model": "Jamba 1.5 Large", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.781820884292191, "raw_score": 0.781820884292191, "raw_scale": 1, "source_record": "Jamba 1.5 Large|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=ai21_jamba-1.5-large,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "b201d0ea15255bb6c9ae", "model": "Gemini 1.5 Flash (001)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7789853346850054, "raw_score": 0.7789853346850054, "raw_scale": 1, "source_record": "Gemini 1.5 Flash (001)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemini-1.5-flash-001,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "e64f38ca25521cb09e85", "model": "Mixtral (8x22B)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7781994679340385, "raw_score": 0.7781994679340385, "raw_scale": 1, "source_record": "Mixtral (8x22B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=mistralai_mixtral-8x22b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "e086206088090bea35d2", "model": "Gemini 1.5 Flash (0514 preview)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7776077742585416, "raw_score": 0.7776077742585416, "raw_scale": 1, "source_record": "Gemini 1.5 Flash (0514 preview)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemini-1.5-flash-preview-0514,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "55bcfac976e077c922cf", "model": "Solar Pro", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7764959772338222, "raw_score": 0.7764959772338222, "raw_scale": 1, "source_record": "Solar Pro|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=upstage_solar-pro-241126,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "600062a50ad07545abc4", "model": "Phi-3 (14B)", "organization": "Microsoft", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.774517567943956, "raw_score": 0.774517567943956, "raw_scale": 1, "source_record": "Phi-3 (14B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=microsoft_phi-3-medium-4k-instruct,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "f68a16785c60bf13e2aa", "model": "Qwen1.5 (72B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7739168514623904, "raw_score": 0.7739168514623904, "raw_scale": 1, "source_record": "Qwen1.5 (72B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=qwen_qwen1.5-72b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "43ac0d5bce97c90d4bd1", "model": "Amazon Nova Lite", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7701970132651439, "raw_score": 0.7701970132651439, "raw_scale": 1, "source_record": "Amazon Nova Lite|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=amazon_nova-lite-v1:0,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "19757ae58efa12b8b42c", "model": "Qwen1.5 Chat (110B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.767842526875325, "raw_score": 0.767842526875325, "raw_scale": 1, "source_record": "Qwen1.5 Chat (110B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=qwen_qwen1.5-110b-chat,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "c2b45ebc30b0e1e7a335", "model": "GPT-4o mini (2024-07-18)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7672266349885158, "raw_score": 0.7672266349885158, "raw_scale": 1, "source_record": "GPT-4o mini (2024-07-18)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=openai_gpt-4o-mini-2024-07-18,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "c77e8400076518107753", "model": "Yi (34B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7623282062667585, "raw_score": 0.7623282062667585, "raw_scale": 1, "source_record": "Yi (34B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=01-ai_yi-34b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "03bfdd988728ee8dce7a", "model": "Claude 3 Sonnet (20240229)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7591643944489032, "raw_score": 0.7591643944489032, "raw_scale": 1, "source_record": "Claude 3 Sonnet (20240229)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=anthropic_claude-3-sonnet-20240229,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "0e966aed6712e79c7d85", "model": "Gemma 2 (27B)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7570433943977638, "raw_score": 0.7570433943977638, "raw_scale": 1, "source_record": "Gemma 2 (27B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemma-2-27b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "dd06659dfcbf7b826aeb", "model": "Phi-3 (7B)", "organization": "Microsoft", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7567775597029733, "raw_score": 0.7567775597029733, "raw_scale": 1, "source_record": "Phi-3 (7B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=microsoft_phi-3-small-8k-instruct,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "b40712c82a6746074f82", "model": "Qwen1.5 (32B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7443285147738198, "raw_score": 0.7443285147738198, "raw_scale": 1, "source_record": "Qwen1.5 (32B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=qwen_qwen1.5-32b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "c582fa582840886156ef", "model": "Claude 3.5 Haiku (20241022)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7431516832098404, "raw_score": 0.7431516832098404, "raw_scale": 1, "source_record": "Claude 3.5 Haiku (20241022)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=anthropic_claude-3-5-haiku-20241022,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "336625ead9ef35a9a41f", "model": "DBRX Instruct", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7410120567563482, "raw_score": 0.7410120567563482, "raw_scale": 1, "source_record": "DBRX Instruct|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=databricks_dbrx-instruct,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "b32b0c493a65ba3fc452", "model": "Gemini 1.5 Flash (002)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7386012759260526, "raw_score": 0.7386012759260526, "raw_scale": 1, "source_record": "Gemini 1.5 Flash (002)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemini-1.5-flash-002,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "e85bbb8cad1a58ebf6bd", "model": "Claude 3 Haiku (20240307)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7379963704791742, "raw_score": 0.7379963704791742, "raw_scale": 1, "source_record": "Claude 3 Haiku (20240307)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=anthropic_claude-3-haiku-20240307,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "94b95207d0a8a99956ec", "model": "Claude 2.1", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7347573300097616, "raw_score": 0.7347573300097616, "raw_scale": 1, "source_record": "Claude 2.1|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=anthropic_claude-2.1,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "f37cd868efbb94e80a0f", "model": "Qwen2.5 Instruct Turbo (7B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7287077954964658, "raw_score": 0.7287077954964658, "raw_scale": 1, "source_record": "Qwen2.5 Instruct Turbo (7B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=qwen_qwen2.5-7b-instruct-turbo,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "33d1c75343af6501eb7c", "model": "DeepSeek LLM Chat (67B)", "organization": "DeepSeek", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.725042055452075, "raw_score": 0.725042055452075, "raw_scale": 1, "source_record": "DeepSeek LLM Chat (67B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=deepseek-ai_deepseek-llm-67b-chat,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "20d66e4326965eaf7293", "model": "Gemma 2 (9B)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7213434388300994, "raw_score": 0.7213434388300994, "raw_scale": 1, "source_record": "Gemma 2 (9B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemma-2-9b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "e4accb3505d3f3181b48", "model": "Mixtral (8x7B 32K seqlen)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7172533474856093, "raw_score": 0.7172533474856093, "raw_scale": 1, "source_record": "Mixtral (8x7B 32K seqlen)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=mistralai_mixtral-8x7b-32kseqlen,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "f396260acb6345bb63fc", "model": "Amazon Nova Micro", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.7076395165491024, "raw_score": 0.7076395165491024, "raw_scale": 1, "source_record": "Amazon Nova Micro|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=amazon_nova-micro-v1:0,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "82bfc2166795f9d84ac6", "model": "Gemini 1.0 Pro (001)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6999548452855813, "raw_score": 0.6999548452855813, "raw_scale": 1, "source_record": "Gemini 1.0 Pro (001)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemini-1.0-pro-001,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "1cb14d13fff06e179bf1", "model": "Jamba 1.5 Mini", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6990912864332881, "raw_score": 0.6990912864332881, "raw_scale": 1, "source_record": "Jamba 1.5 Mini|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=ai21_jamba-1.5-mini,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "72d2d233d7cffce7a488", "model": "Llama 2 (70B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6954627064646919, "raw_score": 0.6954627064646919, "raw_scale": 1, "source_record": "Llama 2 (70B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-2-70b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "726d1d58d2ee75b8aedc", "model": "Command R Plus", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.693531399017232, "raw_score": 0.693531399017232, "raw_scale": 1, "source_record": "Command R Plus|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=cohere_command-r-plus,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "9f51ac74949920d978f9", "model": "PaLM-2 (Bison)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.692092729805366, "raw_score": 0.692092729805366, "raw_scale": 1, "source_record": "PaLM-2 (Bison)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_text-bison@001,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "17fb867f2228103153c4", "model": "GPT-3.5 Turbo (0613)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6888082681118407, "raw_score": 0.6888082681118407, "raw_scale": 1, "source_record": "GPT-3.5 Turbo (0613)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0613,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "a4eef213302251283b17", "model": "Claude Instant 1.2", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6879381069508809, "raw_score": 0.6879381069508809, "raw_scale": 1, "source_record": "Claude Instant 1.2|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=anthropic_claude-instant-1.2,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "db791366bfe1b3c2e6b5", "model": "Mistral Large (2402)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6878312763571534, "raw_score": 0.6878312763571534, "raw_scale": 1, "source_record": "Mistral Large (2402)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=mistralai_mistral-large-2402,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "2077b183f97c852be318", "model": "Mistral Small (2402)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6872982985284851, "raw_score": 0.6872982985284851, "raw_scale": 1, "source_record": "Mistral Small (2402)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=mistralai_mistral-small-2402,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "18649a188f9befeeba0a", "model": "Qwen1.5 (14B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6864380513233806, "raw_score": 0.6864380513233806, "raw_scale": 1, "source_record": "Qwen1.5 (14B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=qwen_qwen1.5-14b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "8b8341cc9fb3afc20c08", "model": "Arctic Instruct", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.676569761285256, "raw_score": 0.676569761285256, "raw_scale": 1, "source_record": "Arctic Instruct|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=snowflake_snowflake-arctic-instruct,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "d456089eb138ba910241", "model": "GPT-3.5 Turbo (0125)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.672681361696075, "raw_score": 0.672681361696075, "raw_scale": 1, "source_record": "GPT-3.5 Turbo (0125)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=openai_gpt-3.5-turbo-0125,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "d1ccb0f1e970a2c303b9", "model": "Llama 3 (8B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6676368046780581, "raw_score": 0.6676368046780581, "raw_scale": 1, "source_record": "Llama 3 (8B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-3-8b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "5b5167751889bcb56a39", "model": "Gemma (7B)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6611956672419421, "raw_score": 0.6611956672419421, "raw_scale": 1, "source_record": "Gemma (7B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=google_gemma-7b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "e818def671d9efe6a81b", "model": "Jamba Instruct", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6588924296974432, "raw_score": 0.6588924296974432, "raw_scale": 1, "source_record": "Jamba Instruct|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=ai21_jamba-instruct,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "18c03bd9db6f86c06d0f", "model": "Mistral NeMo (2402)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6532946316803779, "raw_score": 0.6532946316803779, "raw_scale": 1, "source_record": "Mistral NeMo (2402)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=mistralai_open-mistral-nemo-2407,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "d2684c534588c4b2f174", "model": "Command R", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6520083694568076, "raw_score": 0.6520083694568076, "raw_scale": 1, "source_record": "Command R|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=cohere_command-r,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "7df4d4bc6267a057b050", "model": "Yi (6B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6400594608704304, "raw_score": 0.6400594608704304, "raw_scale": 1, "source_record": "Yi (6B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=01-ai_yi-6b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "d469d67342f2902b9658", "model": "Qwen1.5 (7B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.6257777441280399, "raw_score": 0.6257777441280399, "raw_scale": 1, "source_record": "Qwen1.5 (7B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=qwen_qwen1.5-7b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "0ac30eb3b69db3d39165", "model": "Mistral Instruct v0.3 (7B)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.5991250371958482, "raw_score": 0.5991250371958482, "raw_scale": 1, "source_record": "Mistral Instruct v0.3 (7B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=mistralai_mistral-7b-instruct-v0.3,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "9066f87ecfeac177a7fd", "model": "Mistral v0.1 (7B)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.5657513230766065, "raw_score": 0.5657513230766065, "raw_scale": 1, "source_record": "Mistral v0.1 (7B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=mistralai_mistral-7b-v0.1,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "50683895b0aaf20d3978", "model": "Llama 3.2 Vision Instruct Turbo (11B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.5650818312394074, "raw_score": 0.5650818312394074, "raw_scale": 1, "source_record": "Llama 3.2 Vision Instruct Turbo (11B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-3.2-11b-vision-instruct-turbo,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "93c6fc0fcd1779a28b4a", "model": "Llama 3.1 Instruct Turbo (8B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.5606310290011183, "raw_score": 0.5606310290011183, "raw_scale": 1, "source_record": "Llama 3.1 Instruct Turbo (8B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-3.1-8b-instruct-turbo,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "d0eb22605e642b68c6b6", "model": "Llama 2 (13B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.5541548609350062, "raw_score": 0.5541548609350062, "raw_scale": 1, "source_record": "Llama 2 (13B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-2-13b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" }, { "id": "7cdb004cc3fe4e3638fa", "model": "Llama 2 (7B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "score": 0.4581089294358622, "raw_score": 0.4581089294358622, "raw_scale": 1, "source_record": "Llama 2 (7B)|HELM v1.13.0; 5-shot; 14042 evaluation instances; mean across 57 subjects", "run_spec_names": [ "mmlu:subject=abstract_algebra,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_abstract_algebra", "mmlu:subject=anatomy,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_anatomy", "mmlu:subject=astronomy,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_astronomy", "mmlu:subject=business_ethics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_business_ethics", "mmlu:subject=clinical_knowledge,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_clinical_knowledge", "mmlu:subject=college_biology,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_college_biology", "mmlu:subject=college_chemistry,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_college_chemistry", "mmlu:subject=college_computer_science,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_college_computer_science", "mmlu:subject=college_mathematics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_college_mathematics", "mmlu:subject=college_medicine,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_college_medicine", "mmlu:subject=college_physics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_college_physics", "mmlu:subject=computer_security,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_computer_security", "mmlu:subject=conceptual_physics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_conceptual_physics", "mmlu:subject=econometrics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_econometrics", "mmlu:subject=electrical_engineering,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_electrical_engineering", "mmlu:subject=elementary_mathematics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_elementary_mathematics", "mmlu:subject=formal_logic,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_formal_logic", "mmlu:subject=global_facts,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_global_facts", "mmlu:subject=high_school_biology,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_biology", "mmlu:subject=high_school_chemistry,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_chemistry", "mmlu:subject=high_school_computer_science,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_computer_science", "mmlu:subject=high_school_european_history,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_european_history", "mmlu:subject=high_school_geography,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_geography", "mmlu:subject=high_school_government_and_politics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_government_and_politics", "mmlu:subject=high_school_macroeconomics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_macroeconomics", "mmlu:subject=high_school_mathematics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_mathematics", "mmlu:subject=high_school_microeconomics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_microeconomics", "mmlu:subject=high_school_physics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_physics", "mmlu:subject=high_school_psychology,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_psychology", "mmlu:subject=high_school_statistics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_statistics", "mmlu:subject=high_school_us_history,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_us_history", "mmlu:subject=high_school_world_history,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_high_school_world_history", "mmlu:subject=human_aging,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_human_aging", "mmlu:subject=human_sexuality,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_human_sexuality", "mmlu:subject=international_law,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_international_law", "mmlu:subject=jurisprudence,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_jurisprudence", "mmlu:subject=logical_fallacies,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_logical_fallacies", "mmlu:subject=machine_learning,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_machine_learning", "mmlu:subject=management,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_management", "mmlu:subject=marketing,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_marketing", "mmlu:subject=medical_genetics,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_medical_genetics", "mmlu:subject=miscellaneous,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_miscellaneous", "mmlu:subject=moral_disputes,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_moral_disputes", "mmlu:subject=moral_scenarios,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_moral_scenarios", "mmlu:subject=nutrition,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_nutrition", "mmlu:subject=philosophy,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_philosophy", "mmlu:subject=prehistory,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_prehistory", "mmlu:subject=professional_accounting,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_professional_accounting", "mmlu:subject=professional_law,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_professional_law", "mmlu:subject=professional_medicine,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_professional_medicine", "mmlu:subject=professional_psychology,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_professional_psychology", "mmlu:subject=public_relations,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_public_relations", "mmlu:subject=security_studies,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_security_studies", "mmlu:subject=sociology,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_sociology", "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_us_foreign_policy", "mmlu:subject=virology,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_virology", "mmlu:subject=world_religions,method=multiple_choice_joint,model=meta_llama-2-7b,eval_split=test,groups=mmlu_world_religions" ], "evaluation_count": 14042.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/mmlu/latest/" } ], "public_history": true }, { "id": "gsm8k-helm", "name": "GSM8K (HELM Lite 2025, v1.13.0, 5-shot)", "group": "数学与科学", "unit": "%", "metric": "Final-answer exact match", "protocol": { "en": "HELM Lite v1.13.0 (2025-01-10), five-shot. Final-answer exact match on 1,000 GSM8K test instances.", "zh": "HELM Lite v1.13.0（2025-01-10），五样本。取 1,000 道 GSM8K 测试题的最终答案精确匹配率。" }, "source": { "data_url": "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/gsm.json", "sha256": "8476d2c7fd3e82fab400106fe504fa030cff3fe051765d200bfebce559f74c74", "retrieved_at": "2026-11-02T05:58:57.184813+00:00", "name": "Stanford CRFM / HELM", "url": "https://crfm.stanford.edu/helm/lite/latest/", "kind": "evaluator", "status": "dated_release", "selector": "HELM v1.13.0; Grade School Math; # train=5; EM; 1000 instances per model", "artifacts": [ { "data_url": "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/summary.json", "sha256": "d4eb487e79bcdd7329bdcb10f5116364c2501842fe5e58da6e0bd717f90ee71e", "retrieved_at": "2026-11-01T16:51:20.444801+00:00" } ] }, "results": [ { "id": "66ceefc0b0993a31661a", "model": "Claude 3.5 Sonnet (20241022)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.956, "raw_score": 0.956, "raw_scale": 1, "source_record": "Claude 3.5 Sonnet (20241022)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-3-5-sonnet-20241022" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "1b5a8e49124bac817b5d", "model": "Claude 3.5 Sonnet (20240620)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.949, "raw_score": 0.949, "raw_scale": 1, "source_record": "Claude 3.5 Sonnet (20240620)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-3-5-sonnet-20240620" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "183aae890435d8cef623", "model": "Llama 3.1 Instruct Turbo (405B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.949, "raw_score": 0.949, "raw_scale": 1, "source_record": "Llama 3.1 Instruct Turbo (405B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-3.1-405b-instruct-turbo,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "2089c7a59ef93a1504d1", "model": "Gemini 2.0 Flash (Experimental)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.946, "raw_score": 0.946, "raw_scale": 1, "source_record": "Gemini 2.0 Flash (Experimental)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_gemini-2.0-flash-exp,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "e1e3f07c688fe852a568", "model": "Llama 3.3 Instruct Turbo (70B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.942, "raw_score": 0.942, "raw_scale": 1, "source_record": "Llama 3.3 Instruct Turbo (70B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-3.3-70b-instruct-turbo,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "8a5ca5cb2b385c45aeea", "model": "DeepSeek v3", "organization": "DeepSeek", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.94, "raw_score": 0.94, "raw_scale": 1, "source_record": "DeepSeek v3|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=deepseek-ai_deepseek-v3,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "54161609ca1ac6e19fa4", "model": "Llama 3.1 Instruct Turbo (70B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.938, "raw_score": 0.938, "raw_scale": 1, "source_record": "Llama 3.1 Instruct Turbo (70B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-3.1-70b-instruct-turbo,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "e5ad9b44cce59c9e017a", "model": "Llama 3.2 Vision Instruct Turbo (90B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.936, "raw_score": 0.936, "raw_scale": 1, "source_record": "Llama 3.2 Vision Instruct Turbo (90B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-3.2-90b-vision-instruct-turbo,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "2fde36c7fdf8fe929d60", "model": "GPT-4 (0613)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.932, "raw_score": 0.932, "raw_scale": 1, "source_record": "GPT-4 (0613)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=openai_gpt-4-0613" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "8303dd190e3212af66f4", "model": "Claude 3 Opus (20240229)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.924, "raw_score": 0.924, "raw_scale": 1, "source_record": "Claude 3 Opus (20240229)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-3-opus-20240229" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "75575a898806d71dbb4f", "model": "Qwen2 Instruct (72B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.92, "raw_score": 0.92, "raw_scale": 1, "source_record": "Qwen2 Instruct (72B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=qwen_qwen2-72b-instruct,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "ba86ef54cde713a82627", "model": "Mistral Large 2 (2407)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.912, "raw_score": 0.912, "raw_scale": 1, "source_record": "Mistral Large 2 (2407)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=mistralai_mistral-large-2407,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "7c1385350de6ad947248", "model": "GPT-4o (2024-08-06)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.909, "raw_score": 0.909, "raw_scale": 1, "source_record": "GPT-4o (2024-08-06)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=openai_gpt-4o-2024-08-06,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "b5431ab6a1588f999aef", "model": "Claude 3 Sonnet (20240229)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.907, "raw_score": 0.907, "raw_scale": 1, "source_record": "Claude 3 Sonnet (20240229)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-3-sonnet-20240229" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "6197c9536b76aeb70e3e", "model": "GPT-4o (2024-05-13)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.905, "raw_score": 0.905, "raw_scale": 1, "source_record": "GPT-4o (2024-05-13)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=openai_gpt-4o-2024-05-13,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "1e8598e0870e6601248a", "model": "Palmyra-X-004", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.905, "raw_score": 0.905, "raw_scale": 1, "source_record": "Palmyra-X-004|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=writer_palmyra-x-004,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "5496239d95cbb9469076", "model": "Qwen2.5 Instruct Turbo (72B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.9, "raw_score": 0.9, "raw_scale": 1, "source_record": "Qwen2.5 Instruct Turbo (72B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=qwen_qwen2.5-72b-instruct-turbo,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "d7e3d1302a4cf1406394", "model": "Phi-3 (14B)", "organization": "Microsoft", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.878, "raw_score": 0.878, "raw_scale": 1, "source_record": "Phi-3 (14B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=microsoft_phi-3-medium-4k-instruct" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "a4c89e34c05ee4603738", "model": "Solar Pro", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.871, "raw_score": 0.871, "raw_scale": 1, "source_record": "Solar Pro|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=upstage_solar-pro-241126,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "e92b4dc3113f07aec0ec", "model": "Amazon Nova Pro", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.87, "raw_score": 0.87, "raw_scale": 1, "source_record": "Amazon Nova Pro|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=amazon_nova-pro-v1:0,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "6d499160b685b1729cb8", "model": "Jamba 1.5 Large", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.846, "raw_score": 0.846, "raw_scale": 1, "source_record": "Jamba 1.5 Large|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=ai21_jamba-1.5-large,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "dd6f1c3249e2bc0acae9", "model": "GPT-4o mini (2024-07-18)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.843, "raw_score": 0.843, "raw_scale": 1, "source_record": "GPT-4o mini (2024-07-18)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=openai_gpt-4o-mini-2024-07-18,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "1dc5b02cb5fa722249c7", "model": "Gemini 1.5 Pro (001)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.836, "raw_score": 0.836, "raw_scale": 1, "source_record": "Gemini 1.5 Pro (001)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_gemini-1.5-pro-001,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "94b80e693792b7ff5f85", "model": "PaLM-2 (Unicorn)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.831, "raw_score": 0.831, "raw_scale": 1, "source_record": "PaLM-2 (Unicorn)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_text-unicorn@001" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "dd40d96e7a92c8fb0c00", "model": "Palmyra X V3 (72B)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.831, "raw_score": 0.831, "raw_scale": 1, "source_record": "Palmyra X V3 (72B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=writer_palmyra-x-v3" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "96cc14fa5accf4424d99", "model": "Qwen2.5 Instruct Turbo (7B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.83, "raw_score": 0.83, "raw_scale": 1, "source_record": "Qwen2.5 Instruct Turbo (7B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=qwen_qwen2.5-7b-instruct-turbo,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "56b8eb81dd3315c26f45", "model": "Amazon Nova Lite", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.829, "raw_score": 0.829, "raw_scale": 1, "source_record": "Amazon Nova Lite|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=amazon_nova-lite-v1:0,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "4c04ba64feef01f6eb7d", "model": "GPT-4 Turbo (2024-04-09)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.824, "raw_score": 0.824, "raw_scale": 1, "source_record": "GPT-4 Turbo (2024-04-09)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=openai_gpt-4-turbo-2024-04-09,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "0a0ab97e3750fe495bcd", "model": "Llama 3.2 Vision Instruct Turbo (11B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.823, "raw_score": 0.823, "raw_scale": 1, "source_record": "Llama 3.2 Vision Instruct Turbo (11B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-3.2-11b-vision-instruct-turbo,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "0f6c4755536cc9443d85", "model": "Gemini 1.5 Pro (002)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.817, "raw_score": 0.817, "raw_scale": 1, "source_record": "Gemini 1.5 Pro (002)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_gemini-1.5-pro-002" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "4627dbf9a65d14582cab", "model": "Gemini 1.0 Pro (002)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.816, "raw_score": 0.816, "raw_scale": 1, "source_record": "Gemini 1.0 Pro (002)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_gemini-1.0-pro-002,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "cdfb8327fac2293ee670", "model": "Claude 3.5 Haiku (20241022)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.815, "raw_score": 0.815, "raw_scale": 1, "source_record": "Claude 3.5 Haiku (20241022)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-3-5-haiku-20241022,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "f8178b41fa79018fdb43", "model": "Qwen1.5 Chat (110B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.815, "raw_score": 0.815, "raw_scale": 1, "source_record": "Qwen1.5 Chat (110B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=qwen_qwen1.5-110b-chat,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "dbdcfbfb6c766d43977f", "model": "Gemma 2 Instruct (27B)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.812, "raw_score": 0.812, "raw_scale": 1, "source_record": "Gemma 2 Instruct (27B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_gemma-2-27b-it" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "b05dc003faab6e98ac71", "model": "Llama 3 (70B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.805, "raw_score": 0.805, "raw_scale": 1, "source_record": "Llama 3 (70B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-3-70b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "33c484af13b110a7166f", "model": "Mixtral (8x22B)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.8, "raw_score": 0.8, "raw_scale": 1, "source_record": "Mixtral (8x22B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=mistralai_mixtral-8x22b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "dd3bbe7485807f1bf3b8", "model": "Qwen1.5 (72B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.799, "raw_score": 0.799, "raw_scale": 1, "source_record": "Qwen1.5 (72B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=qwen_qwen1.5-72b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "44cc1c03eca3c42017c9", "model": "Llama 3.1 Instruct Turbo (8B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.798, "raw_score": 0.798, "raw_scale": 1, "source_record": "Llama 3.1 Instruct Turbo (8B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-3.1-8b-instruct-turbo,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "23ed36b04015331dc639", "model": "DeepSeek LLM Chat (67B)", "organization": "DeepSeek", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.795, "raw_score": 0.795, "raw_scale": 1, "source_record": "DeepSeek LLM Chat (67B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=deepseek-ai_deepseek-llm-67b-chat" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "c0dfb2cd4effd0470b05", "model": "Amazon Nova Micro", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.794, "raw_score": 0.794, "raw_scale": 1, "source_record": "Amazon Nova Micro|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=amazon_nova-micro-v1:0,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "cdf7aa5d01810096f420", "model": "Gemini 1.5 Flash (001)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.785, "raw_score": 0.785, "raw_scale": 1, "source_record": "Gemini 1.5 Flash (001)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_gemini-1.5-flash-001,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "60b6080bd07f558b9052", "model": "Claude v1.3", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.784, "raw_score": 0.784, "raw_scale": 1, "source_record": "Claude v1.3|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-v1.3" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "c31e8a1ad6b7eecc4550", "model": "Mistral NeMo (2402)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.782, "raw_score": 0.782, "raw_scale": 1, "source_record": "Mistral NeMo (2402)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=mistralai_open-mistral-nemo-2407,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "a007f837586c1183b170", "model": "Qwen1.5 (32B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.773, "raw_score": 0.773, "raw_scale": 1, "source_record": "Qwen1.5 (32B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=qwen_qwen1.5-32b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "347cdcdc1da50ad8d7f8", "model": "Arctic Instruct", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.768, "raw_score": 0.768, "raw_scale": 1, "source_record": "Arctic Instruct|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=snowflake_snowflake-arctic-instruct,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "6b6cfdea53a122b7b8f6", "model": "Gemma 2 Instruct (9B)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.762, "raw_score": 0.762, "raw_scale": 1, "source_record": "Gemma 2 Instruct (9B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_gemma-2-9b-it,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "1e10e133cc561c53031c", "model": "Command R Plus", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.738, "raw_score": 0.738, "raw_scale": 1, "source_record": "Command R Plus|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=cohere_command-r-plus,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "46df3f42927f2101f1a8", "model": "Palmyra X V2 (33B)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.735, "raw_score": 0.735, "raw_scale": 1, "source_record": "Palmyra X V2 (33B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=writer_palmyra-x-v2" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "59c6ffead4150ffedcbe", "model": "Mistral Small (2402)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.734, "raw_score": 0.734, "raw_scale": 1, "source_record": "Mistral Small (2402)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=mistralai_mistral-small-2402" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "79867825011b218d80ec", "model": "Claude Instant 1.2", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.721, "raw_score": 0.721, "raw_scale": 1, "source_record": "Claude Instant 1.2|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-instant-1.2" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "6ae592a0de54820bb9d2", "model": "Mistral Medium (2312)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.706, "raw_score": 0.706, "raw_scale": 1, "source_record": "Mistral Medium (2312)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=mistralai_mistral-medium-2312" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "7eb645b89723e6e245ea", "model": "Claude 3 Haiku (20240307)", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.699, "raw_score": 0.699, "raw_scale": 1, "source_record": "Claude 3 Haiku (20240307)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-3-haiku-20240307" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "b6642b1f15c5dcf47617", "model": "Mistral Large (2402)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.694, "raw_score": 0.694, "raw_scale": 1, "source_record": "Mistral Large (2402)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=mistralai_mistral-large-2402" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "9595c2cb6af7f91f8dfe", "model": "Qwen1.5 (14B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.693, "raw_score": 0.693, "raw_scale": 1, "source_record": "Qwen1.5 (14B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=qwen_qwen1.5-14b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "159e90af1dc479bc79e9", "model": "Jamba 1.5 Mini", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.691, "raw_score": 0.691, "raw_scale": 1, "source_record": "Jamba 1.5 Mini|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=ai21_jamba-1.5-mini,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "6b83fec1e5afe036d037", "model": "Yi Large (Preview)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.69, "raw_score": 0.69, "raw_scale": 1, "source_record": "Yi Large (Preview)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=01-ai_yi-large-preview,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "e039f775ec074ece22e8", "model": "DBRX Instruct", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.671, "raw_score": 0.671, "raw_scale": 1, "source_record": "DBRX Instruct|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=databricks_dbrx-instruct" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "d56da9260b7090dee042", "model": "Jamba Instruct", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.67, "raw_score": 0.67, "raw_scale": 1, "source_record": "Jamba Instruct|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=ai21_jamba-instruct,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "70c7b6e5f2f739892c13", "model": "GPT-4 Turbo (1106 preview)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.668, "raw_score": 0.668, "raw_scale": 1, "source_record": "GPT-4 Turbo (1106 preview)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=openai_gpt-4-1106-preview" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "96404eafe3299db9ce5e", "model": "Yi (34B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.648, "raw_score": 0.648, "raw_scale": 1, "source_record": "Yi (34B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=01-ai_yi-34b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "e62b506b431cfa2ac017", "model": "Mixtral (8x7B 32K seqlen)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.622, "raw_score": 0.622, "raw_scale": 1, "source_record": "Mixtral (8x7B 32K seqlen)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=mistralai_mixtral-8x7b-32kseqlen" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "1c14eac729e507cd25d5", "model": "GPT-3.5 (text-davinci-003)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.615, "raw_score": 0.615, "raw_scale": 1, "source_record": "GPT-3.5 (text-davinci-003)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=openai_text-davinci-003" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "b49e5e9ad7e7c6953d94", "model": "PaLM-2 (Bison)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.61, "raw_score": 0.61, "raw_scale": 1, "source_record": "PaLM-2 (Bison)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_text-bison@001" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "559c71489b4aedf2f661", "model": "Claude 2.1", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.604, "raw_score": 0.604, "raw_scale": 1, "source_record": "Claude 2.1|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-2.1" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "51c13ccacd74c66f58aa", "model": "Qwen1.5 (7B)", "organization": "Alibaba", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.6, "raw_score": 0.6, "raw_scale": 1, "source_record": "Qwen1.5 (7B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=qwen_qwen1.5-7b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "30730e8742efb16a9ed1", "model": "Claude 2.0", "organization": "Anthropic", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.583, "raw_score": 0.583, "raw_scale": 1, "source_record": "Claude 2.0|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=anthropic_claude-2.0" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "83f20e074ce306afa9bb", "model": "Phi-2", "organization": "Microsoft", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.581, "raw_score": 0.581, "raw_scale": 1, "source_record": "Phi-2|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=microsoft_phi-2" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "5a8e410ce7c62a852c06", "model": "Llama 2 (70B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.567, "raw_score": 0.567, "raw_scale": 1, "source_record": "Llama 2 (70B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-2-70b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "6770db152fb3fdd88480", "model": "Gemma (7B)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.559, "raw_score": 0.559, "raw_scale": 1, "source_record": "Gemma (7B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_gemma-7b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "6b07d86c62fda7c4cb1e", "model": "Command R", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.551, "raw_score": 0.551, "raw_scale": 1, "source_record": "Command R|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=cohere_command-r,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "3b922626df6426ea95e2", "model": "Mistral Instruct v0.3 (7B)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.538, "raw_score": 0.538, "raw_scale": 1, "source_record": "Mistral Instruct v0.3 (7B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=mistralai_mistral-7b-instruct-v0.3,stop=none" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "a1f72f8cbb9deb96bdcb", "model": "GPT-3.5 Turbo (0613)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.501, "raw_score": 0.501, "raw_scale": 1, "source_record": "GPT-3.5 Turbo (0613)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=openai_gpt-3.5-turbo-0613" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "d4e85d6f9748aff245b1", "model": "Llama 3 (8B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.499, "raw_score": 0.499, "raw_scale": 1, "source_record": "Llama 3 (8B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-3-8b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "c52cfbc9305fab329a45", "model": "LLaMA (65B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.489, "raw_score": 0.489, "raw_scale": 1, "source_record": "LLaMA (65B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-65b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "7d6a9880e974f43129b7", "model": "GPT-3.5 (text-davinci-002)", "organization": "OpenAI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.479, "raw_score": 0.479, "raw_scale": 1, "source_record": "GPT-3.5 (text-davinci-002)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=openai_text-davinci-002" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "2b7fff2aecf57d5bc808", "model": "Command", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.452, "raw_score": 0.452, "raw_scale": 1, "source_record": "Command|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=cohere_command" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "7539bac8d78e84fb8f6e", "model": "Mistral v0.1 (7B)", "organization": "Mistral AI", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.377, "raw_score": 0.377, "raw_scale": 1, "source_record": "Mistral v0.1 (7B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=mistralai_mistral-7b-v0.1" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "9232cb2c27b2cc587c17", "model": "Yi (6B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.375, "raw_score": 0.375, "raw_scale": 1, "source_record": "Yi (6B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=01-ai_yi-6b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "c5b98ec3055571cc2222", "model": "Gemini 1.5 Flash (002)", "organization": "Google", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.328, "raw_score": 0.328, "raw_scale": 1, "source_record": "Gemini 1.5 Flash (002)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=google_gemini-1.5-flash-002" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "d402314e3ddf628bc112", "model": "Falcon (40B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.267, "raw_score": 0.267, "raw_scale": 1, "source_record": "Falcon (40B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=tiiuae_falcon-40b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "3e1c95b26d1662933102", "model": "Llama 2 (13B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.266, "raw_score": 0.266, "raw_scale": 1, "source_record": "Llama 2 (13B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-2-13b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "2558c94f98ee48329d9c", "model": "Jurassic-2 Jumbo (178B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.239, "raw_score": 0.239, "raw_scale": 1, "source_record": "Jurassic-2 Jumbo (178B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=ai21_j2-jumbo" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "22251d8bc568c8bd7569", "model": "Jurassic-2 Grande (17B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.159, "raw_score": 0.159, "raw_scale": 1, "source_record": "Jurassic-2 Grande (17B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=ai21_j2-grande" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "87c35ec8a6a0cefb7d29", "model": "Llama 2 (7B)", "organization": "Meta", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.154, "raw_score": 0.154, "raw_scale": 1, "source_record": "Llama 2 (7B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=meta_llama-2-7b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "01ff776521596d23b8b3", "model": "Command Light", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.149, "raw_score": 0.149, "raw_scale": 1, "source_record": "Command Light|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=cohere_command-light" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "e2d34d26eb7ed2b8f79a", "model": "Luminous Supreme (70B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.137, "raw_score": 0.137, "raw_scale": 1, "source_record": "Luminous Supreme (70B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=AlephAlpha_luminous-supreme" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "08fbee3d578eb90b111b", "model": "Luminous Extended (30B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.075, "raw_score": 0.075, "raw_scale": 1, "source_record": "Luminous Extended (30B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=AlephAlpha_luminous-extended" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "4f93450816750380cc6d", "model": "Falcon (7B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.055, "raw_score": 0.055, "raw_scale": 1, "source_record": "Falcon (7B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=tiiuae_falcon-7b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "9fe77525c4b3d0c0ef12", "model": "OLMo (7B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.044, "raw_score": 0.044, "raw_scale": 1, "source_record": "OLMo (7B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=allenai_olmo-7b" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" }, { "id": "e016f75710478431a7b4", "model": "Luminous Base (13B)", "organization": "", "configuration": "HELM v1.13.0; 5-shot; 1000 evaluation instances", "score": 0.028, "raw_score": 0.028, "raw_scale": 1, "source_record": "Luminous Base (13B)|HELM v1.13.0; 5-shot; 1000 evaluation instances", "run_spec_names": [ "gsm:model=AlephAlpha_luminous-base" ], "evaluation_count": 1000.0, "source_date": "2025-01-10", "report_url": "https://crfm.stanford.edu/helm/lite/latest/" } ], "public_history": true }, { "id": "swe-bench-multilingual", "name": "SWE-bench Multilingual", "group": "编程", "unit": "%", "metric": "Resolved", "protocol": { "en": "Multilingual track with one mini-SWE-agent attempt. Each result records the harness version.", "zh": "多语言赛道，mini-SWE-agent 单次尝试。各条成绩保留 Agent 版本。" }, "source": { "name": "SWE-bench", "url": "https://www.swebench.com/", "data_url": "https://www.swebench.com/", "retrieved_at": "2026-11-02T05:57:31.772402+00:00", "sha256": "c862011a4ee1d1a7199fad6ca0905fa5c64f58216f7217ac1eddf37f28788433", "selector": "Multilingual; mini-SWE-agent; System: Attempts - 1; resolved / 100", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "74bd7d3774797a948310", "model": "Gemini 3 Flash", "organization": "Google DeepMind", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.727, "raw_score": 72.7, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_gemini-3-flash", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "1a019e4617121ae57232", "model": "Claude 4.6 Opus", "organization": "Anthropic", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_claude-4-6-opus", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "80409c9380228580d7fb", "model": "Claude 4.5 Opus", "organization": "Anthropic", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.7070000000000001, "raw_score": 70.7, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_claude-4-5-opus", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "895c65bf38c6e418dcf4", "model": "GLM 5", "organization": "Z.ai", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.6970000000000001, "raw_score": 69.7, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_glm-5", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "94750535626c290ab14f", "model": "Gemini 3 Pro", "organization": "Google DeepMind", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.687, "raw_score": 68.7, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_gemini-3-pro", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "2d520f1e48c6a28c122b", "model": "MiniMax M2.5", "organization": "Minimax", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.6829999999999999, "raw_score": 68.3, "raw_scale": 100, "source_record": "20260216_mini-v2.0.0a0_minimax-2-5", "source_date": "2026-02-16", "checked_by_publisher": true }, { "id": "81f0e40c265bb59fb317", "model": "Kimi K2.5", "organization": "Moonshot AI", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.6729999999999999, "raw_score": 67.3, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_kimi-k2-5", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "33053055ef86e3e82706", "model": "Claude 4.5 Sonnet", "organization": "Anthropic", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.67, "raw_score": 67.0, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_claude-4-5-sonnet", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "480ca130174f6f67b0b4", "model": "GPT 5.2 (high)", "organization": "OpenAI", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.667, "raw_score": 66.7, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_gpt-5-2-high", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "1e00985da9c537e161d2", "model": "GPT 5.2 Codex", "organization": "OpenAI", "configuration": "mini-SWE-agent 2.0.0; 1 attempt", "score": 0.6629999999999999, "raw_score": 66.3, "raw_scale": 100, "source_record": "20260220_mini-v2.0.0_gpt-5-2-codex", "source_date": "2026-02-20", "checked_by_publisher": null }, { "id": "dcf83af81afc0aa3dfa0", "model": "Claude 4.5 Haiku", "organization": "Anthropic", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.647, "raw_score": 64.7, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_claude-4-5-haiku", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "5c24cf1daac84a1da84c", "model": "DeepSeek V3.2", "organization": "DeepSeek", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.59, "raw_score": 59.0, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_deepseek-3-2", "source_date": "2026-02-13", "checked_by_publisher": true }, { "id": "5387dc5aea3cf553cf58", "model": "GPT 5 mini", "organization": "OpenAI", "configuration": "mini-SWE-agent 2.0.0a0; 1 attempt", "score": 0.397, "raw_score": 39.7, "raw_scale": 100, "source_record": "20260213_mini-v2.0.0a0_gpt-5-mini", "source_date": "2026-02-13", "checked_by_publisher": true } ] }, { "id": "frontiermath-erdos", "name": "FrontierMath Erdős", "group": "数学与科学", "unit": "%", "metric": "Mean verified solution rate", "protocol": { "en": "68 conjectures that were open in August 2026. A Lean proof or disproof must pass the official verifier. Runs retain the published model and reasoning settings.", "zh": "68 个在 2026 年 8 月仍未解决的猜想。模型提交的 Lean 证明或反证须通过官方验证器。保留各条成绩的模型与推理设置。" }, "source": { "name": "Epoch AI", "url": "https://epoch.ai/benchmarks/frontiermath-erdos", "data_url": "https://epoch.ai/data/benchmark_data.zip", "retrieved_at": "2026-11-02T05:57:32.770697+00:00", "sha256": "e73bfe888530042645844b87d27d8c7c279b3cd02d2d26bf1712f37c45156961", "selector": "frontiermath_erdos.csv: mean_score", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "status": "current_snapshot", "kind": "evaluator" }, "results": [ { "id": "01be73860887b2ad62b7", "model": "gpt-6.1-sol_max", "organization": "OpenAI", "configuration": "", "score": 0.04411764705882353, "raw_score": 0.04411764705882353, "raw_scale": 1, "source_record": "bloom68-alv3junt54xqrybd", "evaluated_at": "2026-10-01T20:26:05.000Z" }, { "id": "aab104d1773df3dcb80c", "model": "gpt-6-astra_max", "organization": "OpenAI", "configuration": "", "score": 0.029411764705882353, "raw_score": 0.029411764705882353, "raw_scale": 1, "source_record": "bloom68-ultima-e1dhs43ykrmakree", "evaluated_at": "2026-08-28T16:33:49.000Z" }, { "id": "984054205896a99e9b0a", "model": "claude-opus-5-5_max", "organization": "Anthropic", "configuration": "", "score": 0.0294, "raw_score": 0.0294, "raw_scale": 1, "source_record": "bloom68-opus55-m1pc5wbymin28jdj", "evaluated_at": "2026-09-24T13:30:00.000Z" }, { "id": "f53972f4e6b08a204db3", "model": "claude-sonnet-5-5_max", "organization": "Anthropic", "configuration": "", "score": 0.029, "raw_score": 0.029, "raw_scale": 1, "source_record": "bloom68-0ov6a268920re3sz", "evaluated_at": "2026-10-08T13:40:19.965Z" }, { "id": "72bf4f791fce7056872b", "model": "gemini-4-argon_high", "organization": "Google DeepMind", "configuration": "", "score": 0.014705882352941176, "raw_score": 0.014705882352941176, "raw_scale": 1, "source_record": "bloom68-nhb6jkzaqrdr7pf8", "evaluated_at": "2026-10-20T17:29:42.000Z" }, { "id": "85b23ec758f8d8def492", "model": "gpt-6-sol_max", "organization": "OpenAI", "configuration": "", "score": 0.014705882352941176, "raw_score": 0.014705882352941176, "raw_scale": 1, "source_record": "bloom68-gpt6sol-jw3kuj7ihascfke9", "evaluated_at": "2026-09-30T09:15:20.000Z" }, { "id": "a1ae114cda0f96f4993f", "model": "claude-fable-5-1_max", "organization": "Anthropic", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "bloom68-fable51-8t9up8nw9358oyrl", "evaluated_at": "2026-09-01T18:56:31.000Z" }, { "id": "e4d9d851393e2280320f", "model": "claude-fable-5_max", "organization": "Anthropic", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "bloom68-fable-vn2l2xoe9902k9rt", "evaluated_at": "2026-08-28T16:33:06.000Z" }, { "id": "2b73cf04f6ac99f36fa1", "model": "claude-opus-5_max", "organization": "Anthropic", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "bloom68-opus5-2joaoeymbc8dk6j4", "evaluated_at": "2026-08-28T18:03:28.000Z" }, { "id": "4ae54f1debf4bc496679", "model": "gemini-3.8-flash_high", "organization": "Google DeepMind", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "bloom68-gemini38f-aojsg6oyu5hcva8s", "evaluated_at": "2026-09-02T17:31:10.000Z" }, { "id": "6e5e841045defa44095f", "model": "gpt-5.5_xhigh", "organization": "OpenAI", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "bloom68-gpt55-4nh07xafb8lf7jqp", "evaluated_at": "2026-08-28T18:05:03.000Z" }, { "id": "264f5ba4127c5c046ba2", "model": "gpt-5.6-sol_max", "organization": "OpenAI", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "bloom68-sol-jx4yy8n4naukwwge", "evaluated_at": "2026-08-28T15:05:09.000Z" }, { "id": "5d4a0cfaa178ef65e542", "model": "grok-4.7_xhigh", "organization": "xAI", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "bloom68-grok47-c934xdlxd6komspn", "evaluated_at": "2026-09-21T23:18:50.000Z" }, { "id": "f99b109db61c45e2fac5", "model": "kimi-k3_max", "organization": "Moonshot", "configuration": "", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "bloom68-kimi-u2zxwufo24l8te4u", "evaluated_at": "2026-08-28T18:03:48.000Z" } ] }, { "id": "longbench-v2-direct", "name": "LongBench v2 (direct)", "group": "指令与长文", "unit": "%", "metric": "Overall direct-answer accuracy", "protocol": { "en": "Overall accuracy with direct answers. Each run retains its published context limit.", "zh": "直接作答的总体准确率。每项结果保留原榜注明的上下文上限。" }, "source": { "name": "LongBench / THUDM", "url": "https://longbench2.github.io/", "data_url": "https://longbench2.github.io/", "retrieved_at": "2026-11-02T05:57:30.886679+00:00", "sha256": "b16f4683a0970694a1aa4fc74155bc61024fa57f98f257b90cc67d54d48d2d35", "selector": "#results: first Overall column (direct answer); exclude Human and Random", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "73fa1148e3fbdadf038e", "model": "MiniMax-Text-01", "organization": "MiniMax", "configuration": "Direct; context 4M", "score": 0.529, "raw_score": 52.9, "raw_scale": 100, "source_record": "MiniMax-Text-01|2025-01-15|Direct", "source_date": "2025-01-15" }, { "id": "9b6d228bfed407b941ac", "model": "GPT-4o (2024-08-06)", "organization": "OpenAI", "configuration": "Direct; context 128k", "score": 0.501, "raw_score": 50.1, "raw_scale": 100, "source_record": "GPT-4o|2024-08-06|Direct", "source_date": "2024-08-06" }, { "id": "5e243bd391fdb138e651", "model": "Gemini-Exp-1206", "organization": "Google", "configuration": "Direct; context 2M", "score": 0.493, "raw_score": 49.3, "raw_scale": 100, "source_record": "Gemini-Exp-1206|2024-12-06|Direct", "source_date": "2024-12-06" }, { "id": "e3ccac91c0d9718531dd", "model": "Gemini-2.0-Flash", "organization": "Google", "configuration": "Direct; context 1M", "score": 0.474, "raw_score": 47.4, "raw_scale": 100, "source_record": "Gemini-2.0-Flash|2024-12-11|Direct", "source_date": "2024-12-11" }, { "id": "3822619ca7c1bd7f96b5", "model": "Qwen3-235B-A22B-Instruct-2507", "organization": "Alibaba", "configuration": "Direct; context 256k", "score": 0.467, "raw_score": 46.7, "raw_scale": 100, "source_record": "Qwen3-235B-A22B-Instruct-2507|2025-07-22|Direct", "source_date": "2025-07-22" }, { "id": "a67840a7ee105617781e", "model": "GPT-4o (2024-11-20)", "organization": "OpenAI", "configuration": "Direct; context 128k", "score": 0.46, "raw_score": 46.0, "raw_scale": 100, "source_record": "GPT-4o|2024-11-20|Direct", "source_date": "2024-11-20" }, { "id": "8b23f35d4153bd360a65", "model": "Kimi-K2-Instruct", "organization": "Moonshot AI", "configuration": "Direct; context 128k", "score": 0.449, "raw_score": 44.9, "raw_scale": 100, "source_record": "Kimi-K2-Instruct|2025-07-11|Direct", "source_date": "2025-07-11" }, { "id": "bf2a6d5ca3ab83c81b8d", "model": "GLM-4-Plus", "organization": "Z.ai & Tsinghua", "configuration": "Direct; context 128k", "score": 0.44299999999999995, "raw_score": 44.3, "raw_scale": 100, "source_record": "GLM-4-Plus|2024-10-11|Direct", "source_date": "2024-10-11" }, { "id": "6c565ab4e2874c542f84", "model": "Qwen2.5-72B", "organization": "Alibaba", "configuration": "Direct; context 128k", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "Qwen2.5-72B|2024-09-19|Direct", "source_date": "2024-09-19" }, { "id": "4dee230ed68365ce1ff0", "model": "Claude 3.5 Sonnet", "organization": "Anthropic", "configuration": "Direct; context 200k", "score": 0.41, "raw_score": 41.0, "raw_scale": 100, "source_record": "Claude 3.5 Sonnet|2024-10-22|Direct", "source_date": "2024-10-22" }, { "id": "0ae6ee0d6c05a01206e0", "model": "Qwen3-235B-A22B", "organization": "Alibaba", "configuration": "Direct; context 128k", "score": 0.40399999999999997, "raw_score": 40.4, "raw_scale": 100, "source_record": "Qwen3-235B-A22B|2025-04-29|Direct", "source_date": "2025-04-29" }, { "id": "184ded61489894bcb68a", "model": "Qwen3-32B", "organization": "Alibaba", "configuration": "Direct; context 128k", "score": 0.38799999999999996, "raw_score": 38.8, "raw_scale": 100, "source_record": "Qwen3-32B|2025-04-29|Direct", "source_date": "2025-04-29" }, { "id": "d1493fe2ed31252869c0", "model": "o1-mini", "organization": "OpenAI", "configuration": "Direct; context 128k", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "o1-mini|2024-09-12|Direct", "source_date": "2024-09-12" }, { "id": "ff611eab6df37e4a17c6", "model": "Mistral Large 24.11", "organization": "Mistral AI", "configuration": "Direct; context 128k", "score": 0.344, "raw_score": 34.4, "raw_scale": 100, "source_record": "Mistral Large 24.11|2024-11-24|Direct", "source_date": "2024-11-24" }, { "id": "cf5fe0c0d96f11dfbf7f", "model": "Qwen3-30B-A3B", "organization": "Alibaba", "configuration": "Direct; context 128k", "score": 0.326, "raw_score": 32.6, "raw_scale": 100, "source_record": "Qwen3-30B-A3B|2025-04-29|Direct", "source_date": "2025-04-29" }, { "id": "cce2abf9e93c9331981c", "model": "Llama 3.1 70B", "organization": "Meta", "configuration": "Direct; context 128k", "score": 0.316, "raw_score": 31.6, "raw_scale": 100, "source_record": "Llama 3.1 70B|2024-07-23|Direct", "source_date": "2024-07-23" }, { "id": "ce85256bd25d718e28b6", "model": "Nemotron 70B", "organization": "Nvidia", "configuration": "Direct; context 128k", "score": 0.31, "raw_score": 31.0, "raw_scale": 100, "source_record": "Nemotron 70B|2024-10-15|Direct", "source_date": "2024-10-15" }, { "id": "a235805f231a849cbb07", "model": "NExtLong 8B", "organization": "CAS", "configuration": "Direct; context 512k", "score": 0.308, "raw_score": 30.8, "raw_scale": 100, "source_record": "NExtLong 8B|2025-01-23|Direct", "source_date": "2025-01-23" }, { "id": "3b4ad423b984ba859666", "model": "GLM-4-9B", "organization": "Z.ai & Tsinghua", "configuration": "Direct; context 128k", "score": 0.302, "raw_score": 30.2, "raw_scale": 100, "source_record": "GLM-4-9B|2024-06-05|Direct", "source_date": "2024-06-05" }, { "id": "c1b9c5ab412768bbb543", "model": "Llama 3.1 8B", "organization": "Meta", "configuration": "Direct; context 128k", "score": 0.3, "raw_score": 30.0, "raw_scale": 100, "source_record": "Llama 3.1 8B|2024-07-23|Direct", "source_date": "2024-07-23" }, { "id": "786569c049dd8166031a", "model": "Qwen2.5-7B", "organization": "Alibaba", "configuration": "Direct; context 128k", "score": 0.3, "raw_score": 30.0, "raw_scale": 100, "source_record": "Qwen2.5-7B|2024-09-19|Direct", "source_date": "2024-09-19" }, { "id": "76ef9801fc907b090db7", "model": "Llama 3.3 70B", "organization": "Meta", "configuration": "Direct; context 128k", "score": 0.298, "raw_score": 29.8, "raw_scale": 100, "source_record": "Llama 3.3 70B|2024-12-06|Direct", "source_date": "2024-12-06" }, { "id": "4525a3042bace8e9070e", "model": "GPT-4o mini", "organization": "OpenAI", "configuration": "Direct; context 128k", "score": 0.293, "raw_score": 29.3, "raw_scale": 100, "source_record": "GPT-4o mini|2024-07-18|Direct", "source_date": "2024-07-18" }, { "id": "1b1fd63c63bb1d53607c", "model": "Command R+", "organization": "Cohere", "configuration": "Direct; context 128k", "score": 0.278, "raw_score": 27.8, "raw_scale": 100, "source_record": "Command R+|2024-08-30|Direct", "source_date": "2024-08-30" }, { "id": "010cdf8fb9cab846503f", "model": "Mistral Large 2", "organization": "Mistral AI", "configuration": "Direct; context 128k", "score": 0.266, "raw_score": 26.6, "raw_scale": 100, "source_record": "Mistral Large 2|2024-07-24|Direct", "source_date": "2024-07-24" } ] }, { "id": "video-mme-with-subs", "name": "Video-MME (with subtitles)", "group": "视觉与视频", "unit": "%", "metric": "Overall accuracy with subtitles", "protocol": { "en": "Overall accuracy with subtitles. Each result retains the source frame-sampling setting.", "zh": "提供字幕时的总体准确率。各模型保留原表中的采样帧设置。" }, "source": { "name": "Video-MME", "url": "https://video-mme.github.io/home_page.html#leaderboard", "data_url": "https://raw.githubusercontent.com/Video-MME/video-mme.github.io/main/home_page.html", "retrieved_at": "2026-11-02T05:57:30.887642+00:00", "sha256": "02b88dcf1268ac331b79a3db7c1bdb1dde1d9f2389f424340ef6082c98adbe6d", "selector": "Overall w subs / 100; frame setting preserved", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "0f37bc8cf971f028b193", "model": "video-SALMONN 2+", "organization": "THU & ByteDance", "configuration": "With subtitles; frames: 10 fps", "score": 0.816, "raw_score": 81.6, "raw_scale": 100, "source_record": "video-SALMONN 2+|With subtitles; frames: 10 fps", "source_date": "2025-09-28" }, { "id": "299226b30b07a16e9aca", "model": "Gemini 1.5 Pro", "organization": "Google", "configuration": "With subtitles; frames: 1/0.5 fps", "score": 0.813, "raw_score": 81.3, "raw_scale": 100, "source_record": "Gemini 1.5 Pro|With subtitles; frames: 1/0.5 fps", "source_date": "2024-06-15" }, { "id": "26131907cdb5865b1957", "model": "AdaReTaKe", "organization": "HIT & Huawei", "configuration": "With subtitles; frames: 1024", "score": 0.7959999999999999, "raw_score": 79.6, "raw_scale": 100, "source_record": "AdaReTaKe|With subtitles; frames: 1024", "source_date": "2025-03-04" }, { "id": "feb14235187f58013aa7", "model": "JT-VL-Chat", "organization": "China Mobile", "configuration": "With subtitles; frames: 1800", "score": 0.7909999999999999, "raw_score": 79.1, "raw_scale": 100, "source_record": "JT-VL-Chat|With subtitles; frames: 1800", "source_date": "2025-07-24" }, { "id": "c9f009b67b34e500f37d", "model": "Qwen2-VL", "organization": "Alibaba", "configuration": "With subtitles; frames: 768", "score": 0.778, "raw_score": 77.8, "raw_scale": 100, "source_record": "Qwen2-VL|With subtitles; frames: 768", "source_date": "2024-08-19" }, { "id": "ba5095b9bd262553a38b", "model": "GPT-4o", "organization": "OpenAI", "configuration": "With subtitles; frames: 384", "score": 0.772, "raw_score": 77.2, "raw_scale": 100, "source_record": "GPT-4o|With subtitles; frames: 384", "source_date": "2024-06-15" }, { "id": "6e937c3fb55c616b9df5", "model": "LLaVA-Video", "organization": "Bytedance & NTU S-Lab", "configuration": "With subtitles; frames: 64", "score": 0.769, "raw_score": 76.9, "raw_scale": 100, "source_record": "LLaVA-Video|With subtitles; frames: 64", "source_date": "2024-08-28" }, { "id": "2b262ec506ea6fca76d3", "model": "Gemini 1.5 Flash", "organization": "Google", "configuration": "With subtitles; frames: 1/0.5 fps", "score": 0.75, "raw_score": 75.0, "raw_scale": 100, "source_record": "Gemini 1.5 Flash|With subtitles; frames: 1/0.5 fps", "source_date": "2024-06-15" }, { "id": "3f5c820fbbc9933c6ab9", "model": "Oryx-1.5", "organization": "THU & Tencent & NTU", "configuration": "With subtitles; frames: 128", "score": 0.7490000000000001, "raw_score": 74.9, "raw_scale": 100, "source_record": "Oryx-1.5|With subtitles; frames: 128", "source_date": "2024-10-21" }, { "id": "e7b7362df3879081f8f8", "model": "InternVL2.5", "organization": "Shanghai AI Lab", "configuration": "With subtitles; frames: 64", "score": 0.74, "raw_score": 74.0, "raw_scale": 100, "source_record": "InternVL2.5|With subtitles; frames: 64", "source_date": "2024-12-13" }, { "id": "757e2fdab77445eb3c07", "model": "Keye-VL-1.5", "organization": "Kwai Keye Team", "configuration": "With subtitles; frames: 64", "score": 0.74, "raw_score": 74.0, "raw_scale": 100, "source_record": "Keye-VL-1.5|With subtitles; frames: 64", "source_date": "2025-08-29" }, { "id": "d770e870545357c1d36c", "model": "TSPO", "organization": "XJTU & TeleAI & USTB", "configuration": "With subtitles; frames: 128", "score": 0.737, "raw_score": 73.7, "raw_scale": 100, "source_record": "TSPO|With subtitles; frames: 128", "source_date": "2025-09-22" }, { "id": "4a07289433b68eec5ae3", "model": "ViLAMP", "organization": "RUC & Ant Group", "configuration": "With subtitles; frames: 1 fps", "score": 0.735, "raw_score": 73.5, "raw_scale": 100, "source_record": "ViLAMP|With subtitles; frames: 1 fps", "source_date": "2025-04-02" }, { "id": "ed3e2b154fc18f500c4b", "model": "Aria", "organization": "Rhymes AI", "configuration": "With subtitles; frames: 256", "score": 0.721, "raw_score": 72.1, "raw_scale": 100, "source_record": "Aria|With subtitles; frames: 256", "source_date": "2024-10-11" }, { "id": "e8ebf02204488f70213b", "model": "Long-VITA", "organization": "Tencent & NJU", "configuration": "With subtitles; frames: 1 fps", "score": 0.72, "raw_score": 72.0, "raw_scale": 100, "source_record": "Long-VITA|With subtitles; frames: 1 fps", "source_date": "2025-07-23" }, { "id": "af3c0a6cb1329436398b", "model": "LinVT", "organization": "Meituan Inc", "configuration": "With subtitles; frames: 120", "score": 0.7170000000000001, "raw_score": 71.7, "raw_scale": 100, "source_record": "LinVT|With subtitles; frames: 120", "source_date": "2025-02-05" }, { "id": "220b842456a8746f0ef8", "model": "TPO", "organization": "Stanford University", "configuration": "With subtitles; frames: 96", "score": 0.715, "raw_score": 71.5, "raw_scale": 100, "source_record": "TPO|With subtitles; frames: 96", "source_date": "2025-01-26" }, { "id": "f537c93f5aaead3af66d", "model": "LiveCC", "organization": "NUS Show Lab & ByteDance", "configuration": "With subtitles; frames: 2 fps", "score": 0.703, "raw_score": 70.3, "raw_scale": 100, "source_record": "LiveCC|With subtitles; frames: 2 fps", "source_date": "2025-04-26" }, { "id": "b24593458016307fdb51", "model": "VideoLLaMA 3", "organization": "Alibaba", "configuration": "With subtitles; frames: 180", "score": 0.703, "raw_score": 70.3, "raw_scale": 100, "source_record": "VideoLLaMA 3|With subtitles; frames: 180", "source_date": "2025-01-24" }, { "id": "80e9e8587f50f9888c57", "model": "NVILA", "organization": "NVIDIA", "configuration": "With subtitles; frames: 1024", "score": 0.7, "raw_score": 70.0, "raw_scale": 100, "source_record": "NVILA|With subtitles; frames: 1024", "source_date": "2024-11-06" }, { "id": "707dde150da5ba6ca0e1", "model": "QuoTA", "organization": "XMU", "configuration": "With subtitles; frames: 64", "score": 0.7, "raw_score": 70.0, "raw_scale": 100, "source_record": "QuoTA|With subtitles; frames: 64", "source_date": "2025-03-04" }, { "id": "08088eae496d122ebbc0", "model": "VideoChat-Flash", "organization": "Shanghai AI Lab & NJU", "configuration": "With subtitles; frames: 512", "score": 0.6970000000000001, "raw_score": 69.7, "raw_scale": 100, "source_record": "VideoChat-Flash|With subtitles; frames: 512", "source_date": "2025-02-06" }, { "id": "3de4d2aa5362a5c132bb", "model": "LLaVA-OneVision", "organization": "Bytedance & NTU S-Lab", "configuration": "With subtitles; frames: 32", "score": 0.696, "raw_score": 69.6, "raw_scale": 100, "source_record": "LLaVA-OneVision|With subtitles; frames: 32", "source_date": "2024-08-08" }, { "id": "6533f798a06ea469587f", "model": "GPT-4o mini", "organization": "OpenAI", "configuration": "With subtitles; frames: 250", "score": 0.6890000000000001, "raw_score": 68.9, "raw_scale": 100, "source_record": "GPT-4o mini|With subtitles; frames: 250", "source_date": "2024-07-21" }, { "id": "c31520b5f29b555d9d68", "model": "ReTaKe", "organization": "HIT & Huawei", "configuration": "With subtitles; frames: 1024", "score": 0.6890000000000001, "raw_score": 68.9, "raw_scale": 100, "source_record": "ReTaKe|With subtitles; frames: 1024", "source_date": "2025-02-05" }, { "id": "e39d5e8348446139ef4f", "model": "ByteVideoLLM", "organization": "Bytedance", "configuration": "With subtitles; frames: 100", "score": 0.688, "raw_score": 68.8, "raw_scale": 100, "source_record": "ByteVideoLLM|With subtitles; frames: 100", "source_date": "2024-10-21" }, { "id": "851685c2179c3274ee8b", "model": "mPLUG-Owl3", "organization": "Alibaba", "configuration": "With subtitles; frames: 128", "score": 0.6809999999999999, "raw_score": 68.1, "raw_scale": 100, "source_record": "mPLUG-Owl3|With subtitles; frames: 128", "source_date": "2024-11-13" }, { "id": "00f53b9b969fcf3f220d", "model": "MiniCPM-o 2.6", "organization": "OpenBMB", "configuration": "With subtitles; frames: 64", "score": 0.679, "raw_score": 67.9, "raw_scale": 100, "source_record": "MiniCPM-o 2.6|With subtitles; frames: 64", "source_date": "2025-01-16" }, { "id": "a1c5dda7e5b62e98961f", "model": "VideoLLaMA 2", "organization": "Alibaba", "configuration": "With subtitles; frames: 32", "score": 0.647, "raw_score": 64.7, "raw_scale": 100, "source_record": "VideoLLaMA 2|With subtitles; frames: 32", "source_date": "2024-08-29" }, { "id": "b0ad9be17b26e332f469", "model": "MiniCPM-V 2.6", "organization": "OpenBMB", "configuration": "With subtitles; frames: 64", "score": 0.637, "raw_score": 63.7, "raw_scale": 100, "source_record": "MiniCPM-V 2.6|With subtitles; frames: 64", "source_date": "2024-08-12" }, { "id": "377c479b97bf8095d8e6", "model": "GPT-4V", "organization": "OpenAI", "configuration": "With subtitles; frames: 10", "score": 0.633, "raw_score": 63.3, "raw_scale": 100, "source_record": "GPT-4V|With subtitles; frames: 10", "source_date": "2024-06-15" }, { "id": "f5d7e7a0e892d26526af", "model": "Claude 3.5 Sonnet", "organization": "Anthropic", "configuration": "With subtitles; frames: 20", "score": 0.629, "raw_score": 62.9, "raw_scale": 100, "source_record": "Claude 3.5 Sonnet|With subtitles; frames: 20", "source_date": "2024-07-30" }, { "id": "568479338ca951f6e4f4", "model": "TimeMarker", "organization": "Meituan", "configuration": "With subtitles; frames: 128", "score": 0.628, "raw_score": 62.8, "raw_scale": 100, "source_record": "TimeMarker|With subtitles; frames: 128", "source_date": "2024-11-07" }, { "id": "2267816ecb2b560584f3", "model": "InternVL2", "organization": "Shanghai AI Lab", "configuration": "With subtitles; frames: 16", "score": 0.624, "raw_score": 62.4, "raw_scale": 100, "source_record": "InternVL2|With subtitles; frames: 16", "source_date": "2024-07-18" }, { "id": "1c8ada125cf7162dc614", "model": "Video-XL", "organization": "SJTU & BAAI", "configuration": "With subtitles; frames: 128", "score": 0.61, "raw_score": 61.0, "raw_scale": 100, "source_record": "Video-XL|With subtitles; frames: 128", "source_date": "2024-10-24" }, { "id": "af4400c775d19a5f256f", "model": "VITA", "organization": "Tencent Youtu Lab & NJU", "configuration": "With subtitles; frames: 32", "score": 0.5920000000000001, "raw_score": 59.2, "raw_scale": 100, "source_record": "VITA|With subtitles; frames: 32", "source_date": "2024-09-08" }, { "id": "05591f923b46e48ac03b", "model": "VITA 1.5", "organization": "Tencent Youtu Lab & NJU", "configuration": "With subtitles; frames: 16", "score": 0.5870000000000001, "raw_score": 58.7, "raw_scale": 100, "source_record": "VITA 1.5|With subtitles; frames: 16", "source_date": "2025-01-22" }, { "id": "653c643583b815c86c3b", "model": "Kangaroo", "organization": "Meituan & UCAS", "configuration": "With subtitles; frames: 64", "score": 0.5760000000000001, "raw_score": 57.6, "raw_scale": 100, "source_record": "Kangaroo|With subtitles; frames: 64", "source_date": "2024-07-23" }, { "id": "39ffc24809bf034291d1", "model": "Video-CCAM", "organization": "QQMM", "configuration": "With subtitles; frames: 96", "score": 0.574, "raw_score": 57.4, "raw_scale": 100, "source_record": "Video-CCAM|With subtitles; frames: 96", "source_date": "2024-07-16" }, { "id": "cd90d03016660b592532", "model": "Long-LLaVA", "organization": "Amazon", "configuration": "With subtitles; frames: 64", "score": 0.5710000000000001, "raw_score": 57.1, "raw_scale": 100, "source_record": "Long-LLaVA|With subtitles; frames: 64", "source_date": "2024-09-09" }, { "id": "a6f2087b923b19def172", "model": "LongVA", "organization": "NTU S-Lab", "configuration": "With subtitles; frames: 128", "score": 0.5429999999999999, "raw_score": 54.3, "raw_scale": 100, "source_record": "LongVA|With subtitles; frames: 128", "source_date": "2024-06-25" }, { "id": "07f9d8027c53ec6c8e1b", "model": "InternVL-Chat-V1.5", "organization": "Shanghai AI Lab", "configuration": "With subtitles; frames: 10", "score": 0.524, "raw_score": 52.4, "raw_scale": 100, "source_record": "InternVL-Chat-V1.5|With subtitles; frames: 10", "source_date": "2024-06-15" }, { "id": "05abc41e5941f3bb90d5", "model": "Qwen-VL-Max", "organization": "Alibaba", "configuration": "With subtitles; frames: 4", "score": 0.512, "raw_score": 51.2, "raw_scale": 100, "source_record": "Qwen-VL-Max|With subtitles; frames: 4", "source_date": "2024-06-15" }, { "id": "d4dc2687b7e94b3a1049", "model": "ShareGemini", "organization": "XMU", "configuration": "With subtitles; frames: 64", "score": 0.479, "raw_score": 47.9, "raw_scale": 100, "source_record": "ShareGemini|With subtitles; frames: 64", "source_date": "2024-06-20" }, { "id": "60fd12c854690435a7c6", "model": "SliME", "organization": "CASIA", "configuration": "With subtitles; frames: 8", "score": 0.47200000000000003, "raw_score": 47.2, "raw_scale": 100, "source_record": "SliME|With subtitles; frames: 8", "source_date": "2024-07-16" }, { "id": "fe875ce11495b4a019f7", "model": "Chat-UniVi-v1.5", "organization": "PKU", "configuration": "With subtitles; frames: 64", "score": 0.45899999999999996, "raw_score": 45.9, "raw_scale": 100, "source_record": "Chat-UniVi-v1.5|With subtitles; frames: 64", "source_date": "2024-06-15" }, { "id": "d4d716f531038bbeaa09", "model": "VideoChat2-Mistral", "organization": "Shanghai AI Lab", "configuration": "With subtitles; frames: 16", "score": 0.43799999999999994, "raw_score": 43.8, "raw_scale": 100, "source_record": "VideoChat2-Mistral|With subtitles; frames: 16", "source_date": "2024-06-15" }, { "id": "caaed13b6b3082ed0140", "model": "ShareGPT4Video", "organization": "Shanghai AI Lab", "configuration": "With subtitles; frames: 16", "score": 0.436, "raw_score": 43.6, "raw_scale": 100, "source_record": "ShareGPT4Video|With subtitles; frames: 16", "source_date": "2024-06-17" }, { "id": "8bec1e1b36d89174494e", "model": "ST-LLM", "organization": "PKU", "configuration": "With subtitles; frames: 64", "score": 0.423, "raw_score": 42.3, "raw_scale": 100, "source_record": "ST-LLM|With subtitles; frames: 64", "source_date": "2024-06-15" }, { "id": "386fa31df8a5a78a4b61", "model": "Qwen-VL-Chat", "organization": "Alibaba", "configuration": "With subtitles; frames: 4", "score": 0.419, "raw_score": 41.9, "raw_scale": 100, "source_record": "Qwen-VL-Chat|With subtitles; frames: 4", "source_date": "2024-06-15" }, { "id": "470df7c43f95b5ba281b", "model": "Video-LLaVA", "organization": "PKU", "configuration": "With subtitles; frames: 8", "score": 0.41600000000000004, "raw_score": 41.6, "raw_scale": 100, "source_record": "Video-LLaVA|With subtitles; frames: 8", "source_date": "2024-06-15" } ] }, { "id": "chinese-simpleqa", "name": "Chinese SimpleQA (2024 paper baselines)", "group": "知识与推理", "unit": "%", "metric": "Correct (CO)", "protocol": { "en": "Original 2024 paper, Table 3. Correct answers across all 3,000 questions (CO), not F-score or accuracy conditional on answering.", "zh": "2024 年原论文表 3 的基线成绩。使用全部 3,000 道题的答对比例 CO，不使用 F-score 或已作答题准确率。" }, "source": { "name": "Chinese SimpleQA authors", "url": "https://arxiv.org/html/2411.07140v2#S3.T3", "data_url": "https://arxiv.org/html/2411.07140v2", "retrieved_at": "2026-11-02T05:57:31.408566+00:00", "sha256": "e1fc402052c92ba493405912869589d01169d24e6b5ea4ad50df2b621d571d6c", "selector": "Original paper Table 3: CO / 100 (not F-score or CGA)", "status": "paper_baselines", "kind": "official_leaderboard" }, "results": [ { "id": "d6bbfa6dd181a9fd16f8", "model": "o1-preview", "organization": "OpenAI", "configuration": "", "score": 0.638, "raw_score": 63.8, "raw_scale": 100, "source_record": "o1-preview" }, { "id": "617fb9bbac15baf9ed46", "model": "Doubao-pro-32k", "organization": "", "configuration": "", "score": 0.619, "raw_score": 61.9, "raw_scale": 100, "source_record": "Doubao-pro-32k" }, { "id": "9b816f4a6295f2b55dd9", "model": "GPT-4o", "organization": "OpenAI", "configuration": "", "score": 0.593, "raw_score": 59.3, "raw_scale": 100, "source_record": "GPT-4o" }, { "id": "1597c5ecedf4619c81f2", "model": "GLM-4-Plus", "organization": "Z AI", "configuration": "", "score": 0.5870000000000001, "raw_score": 58.7, "raw_scale": 100, "source_record": "GLM-4-Plus" }, { "id": "07b233ea6b2765b0ab87", "model": "Gemini-1.5-pro", "organization": "Google", "configuration": "", "score": 0.544, "raw_score": 54.4, "raw_scale": 100, "source_record": "Gemini-1.5-pro" }, { "id": "f21b4930dbaaaa40c17c", "model": "DeepSeek-V2.5", "organization": "DeepSeek", "configuration": "", "score": 0.541, "raw_score": 54.1, "raw_scale": 100, "source_record": "DeepSeek-V2.5" }, { "id": "863dd118e157760796c9", "model": "Qwen-Max", "organization": "Alibaba", "configuration": "", "score": 0.541, "raw_score": 54.1, "raw_scale": 100, "source_record": "Qwen-Max" }, { "id": "16a8f208c319189d09a7", "model": "moonshot-v1-8k", "organization": "Moonshot AI", "configuration": "", "score": 0.48700000000000004, "raw_score": 48.7, "raw_scale": 100, "source_record": "moonshot-v1-8k" }, { "id": "ef968264b3653fc6bc44", "model": "Qwen2.5-72B", "organization": "Alibaba", "configuration": "", "score": 0.484, "raw_score": 48.4, "raw_scale": 100, "source_record": "Qwen2.5-72B" }, { "id": "dbe1e7661453b52708e1", "model": "Yi-Large", "organization": "01.AI", "configuration": "", "score": 0.473, "raw_score": 47.3, "raw_scale": 100, "source_record": "Yi-Large" }, { "id": "e7b8153b595189f22361", "model": "Claude-3.5-Sonnet", "organization": "Anthropic", "configuration": "", "score": 0.462, "raw_score": 46.2, "raw_scale": 100, "source_record": "Claude-3.5-Sonnet" }, { "id": "1e363cbdf3c14b1aaefc", "model": "GPT-4-turbo", "organization": "OpenAI", "configuration": "", "score": 0.456, "raw_score": 45.6, "raw_scale": 100, "source_record": "GPT-4-turbo" }, { "id": "29f11dbc633f60efe5dc", "model": "GPT-4", "organization": "OpenAI", "configuration": "", "score": 0.45399999999999996, "raw_score": 45.4, "raw_scale": 100, "source_record": "GPT-4" }, { "id": "f59eff54e8a6c478e129", "model": "Baichuan3-turbo", "organization": "Baichuan", "configuration": "", "score": 0.452, "raw_score": 45.2, "raw_scale": 100, "source_record": "Baichuan3-turbo" }, { "id": "bc34050dccc424e86f13", "model": "DeepSeek-67B", "organization": "DeepSeek", "configuration": "", "score": 0.435, "raw_score": 43.5, "raw_scale": 100, "source_record": "DeepSeek-67B" }, { "id": "973c6eb0e0af7b3d4ccf", "model": "o1-mini", "organization": "OpenAI", "configuration": "", "score": 0.395, "raw_score": 39.5, "raw_scale": 100, "source_record": "o1-mini" }, { "id": "5ae14ecd7c1956c88930", "model": "Qwen2.5-32B", "organization": "Alibaba", "configuration": "", "score": 0.38799999999999996, "raw_score": 38.8, "raw_scale": 100, "source_record": "Qwen2.5-32B" }, { "id": "a1c8b03618979b98ec34", "model": "LLaMA3.1-70B", "organization": "Meta", "configuration": "", "score": 0.38299999999999995, "raw_score": 38.3, "raw_scale": 100, "source_record": "LLaMA3.1-70B" }, { "id": "0ced3491fe96b78112af", "model": "GPT-4o mini", "organization": "OpenAI", "configuration": "", "score": 0.376, "raw_score": 37.6, "raw_scale": 100, "source_record": "GPT-4o mini" }, { "id": "deb86d494f0fefbdab69", "model": "Doubao-lite-4k", "organization": "", "configuration": "", "score": 0.36700000000000005, "raw_score": 36.7, "raw_scale": 100, "source_record": "Doubao-lite-4k" }, { "id": "ea7711c112b6a38d47e8", "model": "Qwen2.5-14B", "organization": "Alibaba", "configuration": "", "score": 0.354, "raw_score": 35.4, "raw_scale": 100, "source_record": "Qwen2.5-14B" }, { "id": "de7173da042f15cfc73b", "model": "DeepSeek-V2-Lite-Chat", "organization": "DeepSeek", "configuration": "", "score": 0.337, "raw_score": 33.7, "raw_scale": 100, "source_record": "DeepSeek-V2-Lite-Chat" }, { "id": "3245fa945f42dc5dfe6a", "model": "InternLM2.5-20B", "organization": "Shanghai AI Laboratory", "configuration": "", "score": 0.315, "raw_score": 31.5, "raw_scale": 100, "source_record": "InternLM2.5-20B" }, { "id": "fb4a8330a8daf37f7970", "model": "Yi-1.5-34B", "organization": "01.AI", "configuration": "", "score": 0.309, "raw_score": 30.9, "raw_scale": 100, "source_record": "Yi-1.5-34B" }, { "id": "df140e5512e89ff28b21", "model": "GPT-3.5", "organization": "OpenAI", "configuration": "", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "GPT-3.5" }, { "id": "f7e707367b35c8316fa3", "model": "Mixtral-8x22B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "", "score": 0.273, "raw_score": 27.3, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct-v0.1" }, { "id": "9d7101d42cd985023a43", "model": "Qwen2.5-7B", "organization": "Alibaba", "configuration": "", "score": 0.266, "raw_score": 26.6, "raw_scale": 100, "source_record": "Qwen2.5-7B" }, { "id": "e42f7ecbfcd57284414b", "model": "GLM4-9B", "organization": "Z AI", "configuration": "", "score": 0.259, "raw_score": 25.9, "raw_scale": 100, "source_record": "GLM4-9B" }, { "id": "8484f3b1100a157a6878", "model": "InternLM2.5-7B", "organization": "Shanghai AI Laboratory", "configuration": "", "score": 0.247, "raw_score": 24.7, "raw_scale": 100, "source_record": "InternLM2.5-7B" }, { "id": "e48f088e20a1a725df96", "model": "DeepSeek-7B", "organization": "DeepSeek", "configuration": "", "score": 0.23199999999999998, "raw_score": 23.2, "raw_scale": 100, "source_record": "DeepSeek-7B" }, { "id": "0a800dac99fb938f31f2", "model": "Mixtral-8x7B-Instruct-v0.1", "organization": "Mistral AI", "configuration": "", "score": 0.204, "raw_score": 20.4, "raw_scale": 100, "source_record": "Mixtral-8x7B-Instruct-v0.1" }, { "id": "a7cd092f018246f79ee6", "model": "Baichuan2-13B", "organization": "Baichuan", "configuration": "", "score": 0.191, "raw_score": 19.1, "raw_scale": 100, "source_record": "Baichuan2-13B" }, { "id": "ffb1921ef6794e8d0859", "model": "Yi-1.5-9B", "organization": "01.AI", "configuration": "", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "Yi-1.5-9B" }, { "id": "c645b87a973b2a2ba492", "model": "LLaMA3.1-8B", "organization": "Meta", "configuration": "", "score": 0.16899999999999998, "raw_score": 16.9, "raw_scale": 100, "source_record": "LLaMA3.1-8B" }, { "id": "eee056585dc21a7875ba", "model": "Qwen2.5-3B", "organization": "Alibaba", "configuration": "", "score": 0.162, "raw_score": 16.2, "raw_scale": 100, "source_record": "Qwen2.5-3B" }, { "id": "8a19057d0a8f0bacaa03", "model": "Yi-1.5-6B", "organization": "01.AI", "configuration": "", "score": 0.159, "raw_score": 15.9, "raw_scale": 100, "source_record": "Yi-1.5-6B" }, { "id": "c55a8bf0730a3573e29d", "model": "Mistral-7B-Instruct-v0.2", "organization": "Mistral AI", "configuration": "", "score": 0.15, "raw_score": 15.0, "raw_scale": 100, "source_record": "Mistral-7B-Instruct-v0.2" }, { "id": "119e35ff7b48ecf07a23", "model": "Baichuan2-7B", "organization": "Baichuan", "configuration": "", "score": 0.125, "raw_score": 12.5, "raw_scale": 100, "source_record": "Baichuan2-7B" }, { "id": "f562f048c428636754e2", "model": "ChatGLM3-6B", "organization": "Z AI", "configuration": "", "score": 0.11199999999999999, "raw_score": 11.2, "raw_scale": 100, "source_record": "ChatGLM3-6B" }, { "id": "6fe24630ac837333fd73", "model": "Qwen2.5-1.5B", "organization": "Alibaba", "configuration": "", "score": 0.111, "raw_score": 11.1, "raw_scale": 100, "source_record": "Qwen2.5-1.5B" }, { "id": "906d311188af6f5ed88e", "model": "InternLM2.5-1.8B", "organization": "Shanghai AI Laboratory", "configuration": "", "score": 0.053, "raw_score": 5.3, "raw_scale": 100, "source_record": "InternLM2.5-1.8B" } ], "public_history": true }, { "id": "swe-bench-pro-v2-hard", "name": "SWE-bench Pro V2 (Hard)", "group": "编程", "unit": "%", "metric": "Resolved", "protocol": { "en": "Scale V2 Hard subset. Each result keeps the model, agent and reasoning setting published on the board.", "zh": "Scale V2 Hard 子集。各项成绩保留榜单注明的模型、Agent 和推理设置。" }, "source": { "data_url": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "sha256": "b8efacea23d2a9b326009f31d08043d78e5c55a17999f94a9ce354028b9ce819", "retrieved_at": "2026-11-02T05:57:36.682977+00:00", "name": "Scale AI", "url": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2", "selector": "Next page variants[key=hard].entries; exclude deprecated; score / 100", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "3cd46a5a35ec4dea610b", "model": "Opus 5 (Claude Code) xhigh", "organization": "anthropic", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.98, "raw_score": 98.0, "raw_scale": 100, "source_record": "Opus 5 (Claude Code) xhigh|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T15:34:02.000Z", "confidence_interval": 0 }, { "id": "b897078ca5c096756d3b", "model": "Fable 5.1 (Claude Code) high", "organization": "anthropic", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.922, "raw_score": 92.2, "raw_scale": 100, "source_record": "Fable 5.1 (Claude Code) high|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T15:34:45.000Z", "confidence_interval": 0 }, { "id": "220ae1e04d55e5bffbca", "model": "Gemini 4 Argon (mini-swe-agent) high", "organization": "openai", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.9216, "raw_score": 92.16, "raw_scale": 100, "source_record": "Gemini 4 Argon (mini-swe-agent) high|V2 Hard; harness as listed in the model name", "source_date": "2026-10-14T16:34:01.000Z", "confidence_interval": 0 }, { "id": "51f2bee077ccb8d20763", "model": "GPT-6 Astra (Codex) high", "organization": "openai", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "GPT-6 Astra (Codex) high|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T15:35:13.000Z", "confidence_interval": 0 }, { "id": "fd7d04d768d6410dea5a", "model": "Muse Wonder 1.0 (mini-swe-agent) xhigh", "organization": "openai", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "Muse Wonder 1.0 (mini-swe-agent) xhigh|V2 Hard; harness as listed in the model name", "source_date": "2026-10-14T17:57:24.000Z", "confidence_interval": 0 }, { "id": "360ff4c880aca18dbd80", "model": "Kimi-K3 (mini-swe-agent) max", "organization": "moonshot", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.882, "raw_score": 88.2, "raw_scale": 100, "source_record": "Kimi-K3 (mini-swe-agent) max|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T03:41:35.000Z", "confidence_interval": 0 }, { "id": "b28f1bd2c8b93315d805", "model": "Sonnet 5 (Claude Code) xhigh", "organization": "anthropic", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.882, "raw_score": 88.2, "raw_scale": 100, "source_record": "Sonnet 5 (Claude Code) xhigh|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T03:41:34.000Z", "confidence_interval": 0 }, { "id": "80732cbe3c5b73f59ba7", "model": "GPT-5.6 Terra (Codex) xhigh", "organization": "openai", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.863, "raw_score": 86.3, "raw_scale": 100, "source_record": "GPT-5.6 Terra (Codex) xhigh|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T03:41:35.000Z", "confidence_interval": 0 }, { "id": "a7b00765d8be14b1ca54", "model": "GLM-5.3 (mini-swe-agent) max", "organization": "zai", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.843, "raw_score": 84.3, "raw_scale": 100, "source_record": "GLM-5.3 (mini-swe-agent) max|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T03:41:36.000Z", "confidence_interval": 0 }, { "id": "a5e8b12e74c9ba70b59b", "model": "GPT-5.6 Sol (Codex) xhigh", "organization": "openai", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.8240000000000001, "raw_score": 82.4, "raw_scale": 100, "source_record": "GPT-5.6 Sol (Codex) xhigh|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T03:41:36.000Z", "confidence_interval": 0 }, { "id": "e965ad2ebbc2e12c2c62", "model": "Muse Spark Open (mini-swe-agent) xhigh", "organization": "openai", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.8234999999999999, "raw_score": 82.35, "raw_scale": 100, "source_record": "Muse Spark Open (mini-swe-agent) xhigh|V2 Hard; harness as listed in the model name", "source_date": "2026-10-16T19:17:35.000Z", "confidence_interval": 0 }, { "id": "8420c9f32496b1f31729", "model": "Sonnet 5.5 (mini-swe-agent) xhigh", "organization": "anthropic", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.7059000000000001, "raw_score": 70.59, "raw_scale": 100, "source_record": "Sonnet 5.5 (mini-swe-agent) xhigh|V2 Hard; harness as listed in the model name", "source_date": "2026-10-12T23:21:32.000Z", "confidence_interval": 0 }, { "id": "1e0e0789aa8493236fc9", "model": "Gemini 3.8 Flash (mini-swe-agent) high", "organization": "google", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.588, "raw_score": 58.8, "raw_scale": 100, "source_record": "Gemini 3.8 Flash (mini-swe-agent) high|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T03:41:37.000Z", "confidence_interval": 0 }, { "id": "55ac747f416b8c388a0f", "model": "Inkling (mini-swe-agent) xhigh", "organization": "thinkingmachines", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.569, "raw_score": 56.9, "raw_scale": 100, "source_record": "Inkling (mini-swe-agent) xhigh|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T03:41:37.000Z", "confidence_interval": 0 }, { "id": "34d964af0be86b422ac8", "model": "Haiku 5.5 (mini-swe-agent) high", "organization": "anthropic", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.5294, "raw_score": 52.94, "raw_scale": 100, "source_record": "Haiku 5.5 (mini-swe-agent) high|V2 Hard; harness as listed in the model name", "source_date": "2026-10-08T20:32:34.000Z", "confidence_interval": 13.01 }, { "id": "b84ae96691826a0df053", "model": "Haiku 4.5 (Claude Code) xhigh", "organization": "anthropic", "configuration": "V2 Hard; harness as listed in the model name", "score": 0.255, "raw_score": 25.5, "raw_scale": 100, "source_record": "Haiku 4.5 (Claude Code) xhigh|V2 Hard; harness as listed in the model name", "source_date": "2026-09-22T04:26:34.000Z", "confidence_interval": 0 } ] }, { "id": "bigcodebench-hard-complete", "name": "BigCodeBench Hard (Complete)", "group": "编程", "unit": "%", "metric": "Calibrated pass@1", "protocol": { "en": "Hard-Complete, calibrated pass@1 with greedy decoding. Each run retains its prompt and prefilling settings.", "zh": "Hard-Complete，采用贪心解码的校准 pass@1。各条结果保留提示与预填设置。" }, "source": { "data_url": "https://bigcode-bench.github.io/results-hard.json", "sha256": "6c3b7df370bb95e3671aa70979c53a81fc0f946e29f574e3e62b225db8024d63", "retrieved_at": "2026-11-02T05:57:36.498528+00:00", "name": "BigCodeBench", "url": "https://bigcode-bench.github.io/", "selector": "Hard: pass@1.complete; numeric scores including zero", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "f7d79f342a93114cdb2a", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "DeepSeek-R1", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1" }, { "id": "4973eaa8e3ba92b1a901", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "DeepSeek-V3", "source_date": "2024-12-26", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3" }, { "id": "01ef0a20735eb6761f64", "model": "Gemini-Exp-1206", "organization": "Google", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "Gemini-Exp-1206", "source_date": "2024-12-06", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "4fca3357dbc6a3419617", "model": "Claude-3.7-Sonnet-20250219 (temperature=1, length=12800, reasoning=3200)", "organization": "Anthropic", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.392, "raw_score": 39.2, "raw_scale": 100, "source_record": "Claude-3.7-Sonnet-20250219 (temperature=1, length=12800, reasoning=3200)", "source_date": "2025-02-19", "report_url": "https://www.anthropic.com/news/claude-3-7-sonnet" }, { "id": "fcc7bb8c70dc67f1caaa", "model": "Grok-3-Beta", "organization": "xAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.392, "raw_score": 39.2, "raw_scale": 100, "source_record": "Grok-3-Beta", "source_date": "2025-04-11", "report_url": "https://x.ai/news/grok-3" }, { "id": "d469b608278b779018bf", "model": "o1-2024-12-17 (temperature=1, reasoning=low)", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.392, "raw_score": 39.2, "raw_scale": 100, "source_record": "o1-2024-12-17 (temperature=1, reasoning=low)", "source_date": "2024-12-17", "report_url": "https://openai.com/o1/" }, { "id": "b1dad343b8e41526c0f5", "model": "o1-2024-12-17 (temperature=1, reasoning=high)", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.385, "raw_score": 38.5, "raw_scale": 100, "source_record": "o1-2024-12-17 (temperature=1, reasoning=high)", "source_date": "2024-12-17", "report_url": "https://openai.com/o1/" }, { "id": "3a5e1b691f54958de124", "model": "Quasar-Alpha", "organization": "", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "Quasar-Alpha", "source_date": "2025-04-02", "report_url": "https://openrouter.ai/openrouter/quasar-alpha" }, { "id": "67b90159471f802a0870", "model": "o3-mini-2025-01-31 (temperature=1, reasoning=high)", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "o3-mini-2025-01-31 (temperature=1, reasoning=high)", "source_date": "2025-01-31", "report_url": "https://openai.com/index/openai-o3-mini/" }, { "id": "d92b65fc790e8cbbac17", "model": "o3-mini-2025-01-31 (temperature=1, reasoning=low)", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "o3-mini-2025-01-31 (temperature=1, reasoning=low)", "source_date": "2025-01-31", "report_url": "https://openai.com/index/openai-o3-mini/" }, { "id": "8e2b7294bf3d72cb64e2", "model": "o3-mini-2025-01-31 (temperature=1, reasoning=medium)", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "o3-mini-2025-01-31 (temperature=1, reasoning=medium)", "source_date": "2025-01-31", "report_url": "https://openai.com/index/openai-o3-mini/" }, { "id": "737aa6ae38a6f6aca30a", "model": "Athene-V2-Chat", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.37200000000000005, "raw_score": 37.2, "raw_scale": 100, "source_record": "Athene-V2-Chat", "source_date": "2024-11-14", "report_url": "https://huggingface.co/Nexusflow/Athene-V2-Chat" }, { "id": "50e6e7b80a960685f1bf", "model": "o1-2024-12-17 (temperature=1, reasoning=medium)", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.37200000000000005, "raw_score": 37.2, "raw_scale": 100, "source_record": "o1-2024-12-17 (temperature=1, reasoning=medium)", "source_date": "2024-12-17", "report_url": "https://openai.com/o1/" }, { "id": "30e4e0c28d9a83f9eb35", "model": "GPT-4o-2024-08-06", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.365, "raw_score": 36.5, "raw_scale": 100, "source_record": "GPT-4o-2024-08-06", "source_date": "2024-08-06", "report_url": "https://openai.com/index/introducing-structured-outputs-in-the-api/" }, { "id": "39356b04593af94927c7", "model": "Gemini-2.0-Pro-Exp-02-05", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.365, "raw_score": 36.5, "raw_scale": 100, "source_record": "Gemini-2.0-Pro-Exp-02-05", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "29e11ffd28bd1cf04b55", "model": "Gemini-2.5-Pro-Exp-03-25", "organization": "Google", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.365, "raw_score": 36.5, "raw_scale": 100, "source_record": "Gemini-2.5-Pro-Exp-03-25", "source_date": "2025-03-25", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/" }, { "id": "a230375265ad2bb52c0f", "model": "DeepSeek-V3-0324", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.358, "raw_score": 35.8, "raw_scale": 100, "source_record": "DeepSeek-V3-0324", "source_date": "2025-03-24", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3-0324" }, { "id": "77b7de2e585e927cf236", "model": "Qwen2.5-Max", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.358, "raw_score": 35.8, "raw_scale": 100, "source_record": "Qwen2.5-Max", "source_date": "2025-01-28", "report_url": "https://qwenlm.github.io/blog/qwen2.5-max/" }, { "id": "0e9ec69535f16dabc818", "model": "Claude-3.5-Sonnet-20241022", "organization": "Anthropic", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.35100000000000003, "raw_score": 35.1, "raw_scale": 100, "source_record": "Claude-3.5-Sonnet-20241022", "source_date": "2024-10-22", "report_url": "https://claude.ai/" }, { "id": "6c9f0d592b9ed2724c14", "model": "GPT-4-Turbo-2024-04-09", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.35100000000000003, "raw_score": 35.1, "raw_scale": 100, "source_record": "GPT-4-Turbo-2024-04-09", "source_date": "2024-04-09", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "175ebfcba6df18d18674", "model": "Optimus-Alpha", "organization": "", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.35100000000000003, "raw_score": 35.1, "raw_scale": 100, "source_record": "Optimus-Alpha", "source_date": "2025-04-10", "report_url": "https://openrouter.ai/openrouter/optimus-alpha" }, { "id": "a0a9e2fd09d658cbf6f3", "model": "Claude-3.5-Haiku-20241022", "organization": "Anthropic", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "Claude-3.5-Haiku-20241022", "source_date": "2024-10-22", "report_url": "https://claude.ai/" }, { "id": "1327a2256ab190d453a4", "model": "GPT-4o-2024-11-20", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "GPT-4o-2024-11-20", "source_date": "2024-11-20", "report_url": "https://openai.com/gpt-4o/" }, { "id": "e02b40d480ef4ff965b8", "model": "o1-Preview-2024-09-12 (temperature=1)", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "o1-Preview-2024-09-12 (temperature=1)", "source_date": "2024-09-12", "report_url": "https://openai.com/index/introducing-openai-o1-preview/" }, { "id": "e60f64a2394b1789a7dc", "model": "Claude-3.7-Sonnet-20250219", "organization": "Anthropic", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.33799999999999997, "raw_score": 33.8, "raw_scale": 100, "source_record": "Claude-3.7-Sonnet-20250219", "source_date": "2025-02-19", "report_url": "https://www.anthropic.com/news/claude-3-7-sonnet" }, { "id": "05bcc48ff6e5d6a81b45", "model": "GPT-4.1-2025-04-14", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.33799999999999997, "raw_score": 33.8, "raw_scale": 100, "source_record": "GPT-4.1-2025-04-14", "source_date": "2025-04-14", "report_url": "https://openai.com/index/gpt-4-1/" }, { "id": "c331d0495f7d217dc343", "model": "Gemini-2.0-Flash-001", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.33799999999999997, "raw_score": 33.8, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-001", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "b9b4bad7f3f914d4700b", "model": "Gemini-2.0-Flash-Exp", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.33799999999999997, "raw_score": 33.8, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Exp", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "0a1f27c38a7a1134eb67", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.33799999999999997, "raw_score": 33.8, "raw_scale": 100, "source_record": "Qwen2.5-Coder-32B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct" }, { "id": "47956ebc5472be0dba2b", "model": "Athene-V2-Agent", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.331, "raw_score": 33.1, "raw_scale": 100, "source_record": "Athene-V2-Agent", "source_date": "2024-11-14", "report_url": "https://huggingface.co/Nexusflow/Athene-V2-Agent" }, { "id": "36243eb88a6b777df23c", "model": "ChatGPT-4o-latest-20250129", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.331, "raw_score": 33.1, "raw_scale": 100, "source_record": "ChatGPT-4o-latest-20250129", "source_date": "2025-01-29", "report_url": "https://chat.openai.com/" }, { "id": "b662be56ed2b70e20dc0", "model": "Claude-3.5-Sonnet-20240620", "organization": "Anthropic", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.331, "raw_score": 33.1, "raw_scale": 100, "source_record": "Claude-3.5-Sonnet-20240620", "source_date": "2024-06-20", "report_url": "https://claude.ai/" }, { "id": "e2eb3a5beb006734475f", "model": "DeepSeek-Coder-V2-Instruct (2024-07-24)", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.331, "raw_score": 33.1, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct (2024-07-24)", "source_date": "2024-07-24", "report_url": "https://www.deepseek.com/" }, { "id": "28c8e7619670fca9aa09", "model": "DeepSeek-V2-Chat (2024-06-28)", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.324, "raw_score": 32.4, "raw_scale": 100, "source_record": "DeepSeek-V2-Chat (2024-06-28)", "source_date": "2024-06-28", "report_url": "https://www.deepseek.com/" }, { "id": "252556febaf908e90bd9", "model": "Gemini-1.5-Pro-002", "organization": "Google", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.324, "raw_score": 32.4, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-002", "source_date": "2024-09-25", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "f0af06aac84ffa2ce581", "model": "Gemini-Exp-1114", "organization": "Google", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.324, "raw_score": 32.4, "raw_scale": 100, "source_record": "Gemini-Exp-1114", "source_date": "2024-11-14", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "6ef3cfbbec8c41741b3a", "model": "GPT-4.1-Mini-2025-04-14", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "GPT-4.1-Mini-2025-04-14", "source_date": "2025-04-14", "report_url": "https://openai.com/index/gpt-4-1/" }, { "id": "8a90f59cf488785f0113", "model": "GPT-4.1-Nano-2025-04-14", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "GPT-4.1-Nano-2025-04-14", "source_date": "2025-04-14", "report_url": "https://openai.com/index/gpt-4-1/" }, { "id": "5daa6936890763950805", "model": "Gemini-1.5-Pro-Exp-0827", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.318, "raw_score": 31.8, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-Exp-0827", "source_date": "2024-08-27", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "a15e5ef8fcf830c14ff1", "model": "Gemini-1.5-Pro-API-0514", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.311, "raw_score": 31.1, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-API-0514", "source_date": "2024-05-14", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "24cbe07bde6b4936460a", "model": "Gemini-2.0-Flash-Lite-Preview-02-05", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.311, "raw_score": 31.1, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Lite-Preview-02-05", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "74edc4377bcf531b414e", "model": "Grok-3-Mini-Beta (temperature=1, reasoning=low)", "organization": "xAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.311, "raw_score": 31.1, "raw_scale": 100, "source_record": "Grok-3-Mini-Beta (temperature=1, reasoning=low)", "source_date": "2025-04-11", "report_url": "https://x.ai/news/grok-3" }, { "id": "f66cc0642e73ce693351", "model": "Gemini-2.0-Flash-Thinking-Exp-1219", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.304, "raw_score": 30.4, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Thinking-Exp-1219", "source_date": "2024-12-19", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "35a07cb26cdf3628e3a4", "model": "Gemini-Exp-1121", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.304, "raw_score": 30.4, "raw_scale": 100, "source_record": "Gemini-Exp-1121", "source_date": "2024-11-21", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "35bcd780c5bc708f12ee", "model": "Llama-3.1-405B-Instruct", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.304, "raw_score": 30.4, "raw_scale": 100, "source_record": "Llama-3.1-405B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct" }, { "id": "1cec437ecaa9e7f35384", "model": "Phi-4", "organization": "Microsoft", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.304, "raw_score": 30.4, "raw_scale": 100, "source_record": "Phi-4", "source_date": "2024-12-13", "report_url": "https://huggingface.co/microsoft/phi-4" }, { "id": "30b22f7eaedf70c07a8d", "model": "Claude-3-Opus-20240229", "organization": "Anthropic", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Claude-3-Opus-20240229", "source_date": "2024-02-29", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "531cb4f68416b94fca45", "model": "DeepSeek-Coder-V2-Instruct", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct" }, { "id": "3f3248b803702533f6ad", "model": "Gemini-1.5-Pro-Exp-0801", "organization": "Google", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-Exp-0801", "source_date": "2024-08-01", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "5172eb82597f8ce3ceaf", "model": "Gemini-2.0-Flash-Thinking-Exp-01-21", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Thinking-Exp-01-21", "source_date": "2025-01-21", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "008df1b6af1152839d9c", "model": "Grok-3-Mini-Beta (temperature=1, reasoning=high)", "organization": "xAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Grok-3-Mini-Beta (temperature=1, reasoning=high)", "source_date": "2025-04-11", "report_url": "https://x.ai/news/grok-3" }, { "id": "772173e1af502d9b4b99", "model": "Grok-Beta", "organization": "xAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Grok-Beta", "source_date": "2024-03-17", "report_url": "https://grok.com/" }, { "id": "d9be016a0671b4df8089", "model": "Mistral-Large-Instruct-2407", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Mistral-Large-Instruct-2407", "source_date": "2024-07-24", "report_url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407" }, { "id": "c48068679d990e503b8f", "model": "Qwen2.5-Coder-14B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Qwen2.5-Coder-14B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct" }, { "id": "4a32de71d2358d01473e", "model": "Sky-T1-32B-Preview", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.297, "raw_score": 29.7, "raw_scale": 100, "source_record": "Sky-T1-32B-Preview", "source_date": "2025-01-12", "report_url": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Preview" }, { "id": "0ba9009f78dccd01f469", "model": "DeepSeek-R1-Distill-Qwen-32B", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-32B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" }, { "id": "f21b4930dbaaaa40c17c", "model": "DeepSeek-V2.5", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "DeepSeek-V2.5", "source_date": "2024-09-18", "report_url": "https://www.deepseek.com/" }, { "id": "79448501633ce100dfdc", "model": "GPT-4o-2024-05-13", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "GPT-4o-2024-05-13", "source_date": "2024-05-13", "report_url": "https://openai.com/index/hello-gpt-4o/" }, { "id": "41d099c2942e37049603", "model": "Llama-3-70B-Synthia-v3.5", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "Llama-3-70B-Synthia-v3.5", "source_date": "2024-05-27", "report_url": "https://huggingface.co/migtissera/Llama-3-70B-Synthia-v3.5" }, { "id": "48b9f4b337c8e379a4d4", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "Llama-4-Maverick", "source_date": "2025-04-05", "report_url": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct" }, { "id": "972a302e182b812fc2ec", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "Qwen2.5-72B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct" }, { "id": "f3313792cf5138dd4ea3", "model": "Hermes-2-Pro-Llama-3-70B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "Hermes-2-Pro-Llama-3-70B", "source_date": "2024-06-27", "report_url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-70B" }, { "id": "1c5251e09c83d24fbea0", "model": "Llama-3.1-Nemotron-70B-Instruct", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "Llama-3.1-Nemotron-70B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF" }, { "id": "97d06434c085ed09497b", "model": "Llama-3.3-70B-Instruct", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "Llama-3.3-70B-Instruct", "source_date": "2024-12-19", "report_url": "https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct" }, { "id": "b21dbf0e06c7cc54d397", "model": "QwQ-32B (w/ Reasoning)", "organization": "", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "QwQ-32B (w/ Reasoning)", "source_date": "2025-03-06", "report_url": "https://huggingface.co/Qwen/QwQ-32B" }, { "id": "f0311033bb98d0757277", "model": "QwQ-32B-Preview", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "QwQ-32B-Preview", "source_date": "2024-11-28", "report_url": "https://huggingface.co/Qwen/QwQ-32B-Preview" }, { "id": "995765c13d4f27521c03", "model": "Dracarys-Llama-3.1-70B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "Dracarys-Llama-3.1-70B-Instruct", "source_date": "2024-04-23", "report_url": "https://huggingface.co/abacusai/Dracarys-Llama-3.1-70B-Instruct" }, { "id": "55daabb935f5211c65ce", "model": "Llama-3.1-70B-Instruct", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "Llama-3.1-70B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct" }, { "id": "466c60dae478cf162b3d", "model": "Sky-T1-32B-Flash", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "Sky-T1-32B-Flash", "source_date": "2025-01-12", "report_url": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Flash" }, { "id": "1efbd962693289b4c69f", "model": "Dracarys-72B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "Dracarys-72B-Instruct", "source_date": "2024-04-23", "report_url": "https://huggingface.co/abacusai/Dracarys-72B-Instruct" }, { "id": "583c1909ceca9bf51b86", "model": "GPT-4o-mini-2024-07-18", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "GPT-4o-mini-2024-07-18", "source_date": "2024-07-18", "report_url": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/" }, { "id": "6ab0c7f3af37f5d93de4", "model": "Gemma-3-27B-Instruct", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "Gemma-3-27B-Instruct", "source_date": "2025-03-12", "report_url": "https://huggingface.co/google/gemma-3-27b-it" }, { "id": "191e36d7c9e59ab973b8", "model": "Llama-3-70B-Instruct", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "Llama-3-70B-Instruct", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct" }, { "id": "b9b6acf62918ca57f5d9", "model": "Qwen2.5-32B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "Qwen2.5-32B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct" }, { "id": "ae3e8b65664983047b75", "model": "o1-Mini-2024-09-12 (temperature=1)", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "o1-Mini-2024-09-12 (temperature=1)", "source_date": "2024-09-12", "report_url": "https://openai.com/index/introducing-openai-o1-preview/" }, { "id": "88699baaa38a9adf9be3", "model": "Claude-3-Sonnet-20240229", "organization": "Anthropic", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.264, "raw_score": 26.4, "raw_scale": 100, "source_record": "Claude-3-Sonnet-20240229", "source_date": "2024-02-29", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "3baa03ec31c5ad034912", "model": "Phi-3.1-Mini-128K-Instruct", "organization": "Microsoft", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.264, "raw_score": 26.4, "raw_scale": 100, "source_record": "Phi-3.1-Mini-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct" }, { "id": "ce163386751f026ab2c1", "model": "Qwen2-72B-Chat", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.264, "raw_score": 26.4, "raw_scale": 100, "source_record": "Qwen2-72B-Chat", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-72B-Chat" }, { "id": "09151d438cdc834ab3f3", "model": "DeepSeek-V2.5-1210", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.257, "raw_score": 25.7, "raw_scale": 100, "source_record": "DeepSeek-V2.5-1210", "source_date": "2024-12-10" }, { "id": "a3bf0e562269986eeb52", "model": "Gemini-1.5-Flash-API-0514", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "Gemini-1.5-Flash-API-0514", "source_date": "2024-05-14", "report_url": "https://deepmind.google/technologies/gemini/flash/" }, { "id": "b452697db8b6e46ac420", "model": "Athene-70B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.243, "raw_score": 24.3, "raw_scale": 100, "source_record": "Athene-70B", "source_date": "2024-07-20", "report_url": "https://huggingface.co/Nexusflow/Athene-70B" }, { "id": "1a2aca7e452afb3aeff3", "model": "Codestral-22B-v0.1", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.243, "raw_score": 24.3, "raw_scale": 100, "source_record": "Codestral-22B-v0.1", "source_date": "2024-05-23", "report_url": "https://huggingface.co/mistralai/Codestral-22B-v0.1" }, { "id": "aabe2216f3ad8fe8bf24", "model": "Hermes-2-Theta-Llama-3-70B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.243, "raw_score": 24.3, "raw_scale": 100, "source_record": "Hermes-2-Theta-Llama-3-70B", "source_date": "2024-06-24", "report_url": "https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-70B" }, { "id": "28557ddb6a6f5e08a2bd", "model": "Gemini-1.5-Flash-Exp-0827", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "Gemini-1.5-Flash-Exp-0827", "source_date": "2024-08-27", "report_url": "https://deepmind.google/technologies/gemini/flash/" }, { "id": "78dd22077aaf3bb7c874", "model": "KwaiCoder-23B-A4B-v1", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.23600000000000002, "raw_score": 23.6, "raw_scale": 100, "source_record": "KwaiCoder-23B-A4B-v1", "source_date": "2025-01-25", "report_url": "https://huggingface.co/Kwaipilot/KwaiCoder-23B-A4B-v1" }, { "id": "9fa936c7f7eef438953b", "model": "DeepCoder-14B-Preview", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "DeepCoder-14B-Preview", "source_date": "2025-04-09", "report_url": "https://huggingface.co/agentica-org/DeepCoder-14B-Preview" }, { "id": "0556e68fa458808a52fb", "model": "DeepCoder-14B-Preview (w/ Reasoning, 64k tokens, temperature=0.6)", "organization": "", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "DeepCoder-14B-Preview (w/ Reasoning, 64k tokens, temperature=0.6)", "source_date": "2025-04-09", "report_url": "https://huggingface.co/agentica-org/DeepCoder-14B-Preview" }, { "id": "31db15521e03453ab249", "model": "Gemma-2-27B-Instruct", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.223, "raw_score": 22.3, "raw_scale": 100, "source_record": "Gemma-2-27B-Instruct", "source_date": "2024-06-19", "report_url": "https://huggingface.co/google/gemma-2-27b-it" }, { "id": "7020a0f966a0ef1931fb", "model": "GPT-3.5-Turbo-0125", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.21600000000000003, "raw_score": 21.6, "raw_scale": 100, "source_record": "GPT-3.5-Turbo-0125", "source_date": "2024-01-25", "report_url": "https://openai.com/index/new-embedding-models-and-api-updates" }, { "id": "5dd235c408a54c2143af", "model": "Mistral-Nemo-12B-Instruct", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.21600000000000003, "raw_score": 21.6, "raw_scale": 100, "source_record": "Mistral-Nemo-12B-Instruct", "source_date": "2024-07-18", "report_url": "https://huggingface.co/nv-mistralai/Mistral-Nemo-12B-Instruct" }, { "id": "880826db1a9c61240b8d", "model": "DeepSeek-Coder-33B-Instruct", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct" }, { "id": "08683bbc96ba70cd61e3", "model": "DeepSeek-R1-Distill-Qwen-14B", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-14B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B" }, { "id": "df56bf427cd5a73f58d8", "model": "Mixtral-8x22B-Instruct", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct", "source_date": "2024-04-17", "report_url": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1" }, { "id": "56928ca9ba87a6db3cc6", "model": "OpenCodeInterpreter-DS-33B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-33B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B" }, { "id": "2039b67a32ea922cfcd5", "model": "Qwen2.5-14B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "Qwen2.5-14B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct" }, { "id": "2d906b4023f42a6866ad", "model": "WhiteRabbitNeo-33B-v1.5", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.209, "raw_score": 20.9, "raw_scale": 100, "source_record": "WhiteRabbitNeo-33B-v1.5", "source_date": "2024-02-10", "report_url": "https://huggingface.co/WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5" }, { "id": "4a114af99c8d3b80e971", "model": "DeepSeek-R1-Distill-Llama-70B", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.203, "raw_score": 20.3, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Llama-70B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B" }, { "id": "a208d7a5691d17f7397a", "model": "GPT-4-0613", "organization": "OpenAI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.203, "raw_score": 20.3, "raw_scale": 100, "source_record": "GPT-4-0613", "source_date": "2024-06-13", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "d01d5d37cd200a83a4d1", "model": "Qwen2.5-Coder-7B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.203, "raw_score": 20.3, "raw_scale": 100, "source_record": "Qwen2.5-Coder-7B-Instruct", "source_date": "2024-11-12", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct" }, { "id": "82ee262512a54a309c96", "model": "Claude-3-Haiku-20240307", "organization": "Anthropic", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.196, "raw_score": 19.6, "raw_scale": 100, "source_record": "Claude-3-Haiku-20240307", "source_date": "2024-03-07", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "f3dde70fbdce9fea442f", "model": "DeepSeek-Coder-33B-Base", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.196, "raw_score": 19.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-Base", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-base" }, { "id": "9bf57ffa8a433335d160", "model": "Mistral-Small-24B-Instruct-2501", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.196, "raw_score": 19.6, "raw_scale": 100, "source_record": "Mistral-Small-24B-Instruct-2501", "source_date": "2025-01-31", "report_url": "https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501" }, { "id": "3a8fc11881e03b27a72f", "model": "Qwen1.5-110B-Chat", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.196, "raw_score": 19.6, "raw_scale": 100, "source_record": "Qwen1.5-110B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-110B-Chat" }, { "id": "dbe1e7661453b52708e1", "model": "Yi-Large", "organization": "01.AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.196, "raw_score": 19.6, "raw_scale": 100, "source_record": "Yi-Large", "source_date": "2024-05-13", "report_url": "https://www.lingyiwanwu.com/" }, { "id": "dd07a3eb0ca36bde1336", "model": "Mistral-Small-Instruct-2409", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.18899999999999997, "raw_score": 18.9, "raw_scale": 100, "source_record": "Mistral-Small-Instruct-2409", "source_date": "2024-09-18", "report_url": "https://huggingface.co/mistralai/Mistral-Small-Instruct-2409" }, { "id": "691f84511d0ec15341c5", "model": "OpenCoder-8B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.18899999999999997, "raw_score": 18.9, "raw_scale": 100, "source_record": "OpenCoder-8B-Instruct", "source_date": "2024-11-09", "report_url": "https://huggingface.co/infly/OpenCoder-8B-Instruct" }, { "id": "e15aaa25c73230d63687", "model": "ReflectionCoder-DS-6.7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.18899999999999997, "raw_score": 18.9, "raw_scale": 100, "source_record": "ReflectionCoder-DS-6.7B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-6.7B" }, { "id": "eb2e685465bc054d6df0", "model": "AutoCoder", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "AutoCoder", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder" }, { "id": "25841a38654c0d8cc1fa", "model": "DeepSeek-Coder-V2-Lite-Instruct", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Lite-Instruct", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct" }, { "id": "94f34ece2cd608881244", "model": "ReflectionCoder-DS-33B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "ReflectionCoder-DS-33B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-33B" }, { "id": "f78d9deb8a19887b291a", "model": "Tess-v2.5.2-Qwen2-72B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "Tess-v2.5.2-Qwen2-72B", "source_date": "2024-07-18", "report_url": "https://huggingface.co/migtissera/Tess-v2.5.2-Qwen2-72B" }, { "id": "93c7acd979f4b385ae10", "model": "Yi-Coder-9B-Chat", "organization": "01.AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.17600000000000002, "raw_score": 17.6, "raw_scale": 100, "source_record": "Yi-Coder-9B-Chat", "source_date": "2024-09-04", "report_url": "https://huggingface.co/01-ai/Yi-Coder-9B-Chat" }, { "id": "fb257208509ac862fac3", "model": "Llama-4-Scout", "organization": "Meta", "configuration": "Hard-Complete; prefill=False; prompted=True", "score": 0.16899999999999998, "raw_score": 16.9, "raw_scale": 100, "source_record": "Llama-4-Scout", "source_date": "2025-04-05", "report_url": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct" }, { "id": "fb1399a69666a126eaff", "model": "OpenCodeInterpreter-DS-6.7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.16899999999999998, "raw_score": 16.9, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-6.7B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B" }, { "id": "172fc6a41d0f943afeb2", "model": "WaveCoder-Ultra-6.7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.16899999999999998, "raw_score": 16.9, "raw_scale": 100, "source_record": "WaveCoder-Ultra-6.7B", "source_date": "2023-12-26", "report_url": "https://huggingface.co/microsoft/wavecoder-ultra-6.7b" }, { "id": "39651f4435d5050c9212", "model": "Artigenz-Coder-DS-6.7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.162, "raw_score": 16.2, "raw_scale": 100, "source_record": "Artigenz-Coder-DS-6.7B", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Artigenz/Artigenz-Coder-DS-6.7B" }, { "id": "50071bd2dc1398fc8866", "model": "CodeLlama-70B-Base", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.162, "raw_score": 16.2, "raw_scale": 100, "source_record": "CodeLlama-70B-Base", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-70b-hf" }, { "id": "a06e8815c7b78da345b5", "model": "Nxcode-CQ-7B-Orpo", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.162, "raw_score": 16.2, "raw_scale": 100, "source_record": "Nxcode-CQ-7B-Orpo", "source_date": "2024-04-25", "report_url": "https://huggingface.co/NTQAI/Nxcode-CQ-7B-orpo" }, { "id": "b61019fb4a8a8670e461", "model": "Qwen2.5-7B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.162, "raw_score": 16.2, "raw_scale": 100, "source_record": "Qwen2.5-7B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct" }, { "id": "457d7f85397735e8f881", "model": "CodeGeex4-All-9B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "CodeGeex4-All-9B", "source_date": "2024-07-05", "report_url": "https://huggingface.co/THUDM/codegeex4-all-9b" }, { "id": "b8f265c234cd3f272e73", "model": "CodeLlama-70B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "CodeLlama-70B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf" }, { "id": "bb051df20f0935ee2e27", "model": "CodeQwen1.5-7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "CodeQwen1.5-7B", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Qwen/CodeQwen1.5-7B" }, { "id": "db1c1e11f754025a1f92", "model": "CodeQwen1.5-7B-Chat", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "CodeQwen1.5-7B-Chat", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat" }, { "id": "584fc8797f1014af0fda", "model": "DeepSeek-Coder-6.7B-Instruct", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct" }, { "id": "0ffe809b06eb35a62f7d", "model": "DeepSeek-V2-Chat", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "DeepSeek-V2-Chat", "source_date": "2024-04-06", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat" }, { "id": "b5bfc24428b007ff6475", "model": "Granite-Code-34B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "Granite-Code-34B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-34b-code-instruct" }, { "id": "21dadbca66e9a80cac7b", "model": "Phi-3-Medium-128K-Instruct", "organization": "Microsoft", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "Phi-3-Medium-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-medium-128k-instruct" }, { "id": "c98656b9280aa90ed5ab", "model": "Phind-CodeLlama-34B-v2", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "Phind-CodeLlama-34B-v2", "source_date": "2023-08-25", "report_url": "https://huggingface.co/Phind/Phind-CodeLlama-34B-v2" }, { "id": "e19fcc044ca2daf86af7", "model": "C4AI-Command-R-Plus-08-2024", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "C4AI-Command-R-Plus-08-2024", "source_date": "2024-08-30", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024" }, { "id": "fc4539651aa3cda0fbdc", "model": "Codestral-Mamba", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "Codestral-Mamba", "source_date": "2024-07-16", "report_url": "https://huggingface.co/mistralai/mamba-codestral-7B-v0.1" }, { "id": "b936f565b01b32eb6fd1", "model": "InverseCoder-CL-13B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "InverseCoder-CL-13B", "source_date": "2024-07-08", "report_url": "https://huggingface.co/wyt2000/InverseCoder-CL-13B" }, { "id": "4ee69372c7e9a2209ddc", "model": "InverseCoder-DS-6.7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "InverseCoder-DS-6.7B", "source_date": "2024-07-08", "report_url": "https://huggingface.co/wyt2000/InverseCoder-DS-6.7B" }, { "id": "883f1f2bbff22c83d490", "model": "Llama-3-70B-Base", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "Llama-3-70B-Base", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-70B" }, { "id": "0667522f2464c8f617db", "model": "Mistral-Small-2402", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "Mistral-Small-2402", "source_date": "2024-02-26", "report_url": "https://mistral.ai/news/mistral-large/" }, { "id": "80b8ad04ff5542d82114", "model": "Qwen2-57B-A14B", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "Qwen2-57B-A14B", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-57B-A14B" }, { "id": "cd6f07fbcf06d98a1a81", "model": "ReflectionCoder-CL-34B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "ReflectionCoder-CL-34B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-34B" }, { "id": "1d2fea03400612233f90", "model": "StarCoder2-15B-Instruct-v0.1", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.149, "raw_score": 14.9, "raw_scale": 100, "source_record": "StarCoder2-15B-Instruct-v0.1", "source_date": "2024-04-30", "report_url": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1" }, { "id": "505064d9efe3613d0dbb", "model": "Command R+", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "Command R+", "source_date": "2024-04-04", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-plus" }, { "id": "e532aaf956fe2b5398af", "model": "Mixtral-8x22B-Base", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "Mixtral-8x22B-Base", "source_date": "2024-04-17", "report_url": "https://huggingface.co/mistralai/Mixtral-8x22B-v0.1" }, { "id": "25647cdb7148cc992141", "model": "OpenChat-3.6-8B-20240522", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "OpenChat-3.6-8B-20240522", "source_date": "2024-05-22", "report_url": "https://huggingface.co/openchat/openchat-3.6-8b-20240522" }, { "id": "4a209c72393be25357eb", "model": "Phi-3-Mini-128K-Instruct", "organization": "Microsoft", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "Phi-3-Mini-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct" }, { "id": "12f23f959927b2011e48", "model": "Phi-3-Small-128K-Instruct", "organization": "Microsoft", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.142, "raw_score": 14.2, "raw_scale": 100, "source_record": "Phi-3-Small-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-small-128k-instruct" }, { "id": "0424a91abbd6ada2af5b", "model": "C4AI-Command-R-08-2024", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "C4AI-Command-R-08-2024", "source_date": "2024-08-30", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-08-2024" }, { "id": "18d08a65d81ce38b6d84", "model": "CodeGemma-7B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "CodeGemma-7B-Instruct", "source_date": "2024-04-10", "report_url": "https://huggingface.co/google/codegemma-7b-it" }, { "id": "10fc70e5dc28194ef284", "model": "DeepSeek-Coder-6.7B-Base", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-Base", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base" }, { "id": "f9b45bd93750ea1d1681", "model": "Llama-3.1-8B-Instruct", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "Llama-3.1-8B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct" }, { "id": "e6e26b4b9e2582b38680", "model": "Qwen2-7B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.135, "raw_score": 13.5, "raw_scale": 100, "source_record": "Qwen2-7B-Instruct", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-7B-Instruct" }, { "id": "f88cc9c0f66d0b1d4001", "model": "AutoCoder-QW-7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "AutoCoder-QW-7B", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder_QW_7B" }, { "id": "cfebb5ea5cad76711273", "model": "CodeLlama-34B-Base", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "CodeLlama-34B-Base", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-34b-hf" }, { "id": "3cdc2c68a81d341ccbae", "model": "InverseCoder-CL-7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "InverseCoder-CL-7B", "source_date": "2024-07-08", "report_url": "https://huggingface.co/wyt2000/InverseCoder-CL-7B" }, { "id": "d1ef871a823b03b69612", "model": "Magicoder-S-DS-6.7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "Magicoder-S-DS-6.7B", "source_date": "2024-12-04", "report_url": "https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B" }, { "id": "3f74609b2e0f679e0594", "model": "Mistral-Large-2402", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "Mistral-Large-2402", "source_date": "2024-02-26", "report_url": "https://mistral.ai/news/mistral-large/" }, { "id": "d1dafd75f9b06dcff96e", "model": "Phi-3.5-Mini-Instruct", "organization": "Microsoft", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "Phi-3.5-Mini-Instruct", "source_date": "2024-08-21", "report_url": "https://huggingface.co/microsoft/Phi-3.5-mini-instruct" }, { "id": "d34c081bee7a70214e79", "model": "Qwen1.5-32B-Chat", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.128, "raw_score": 12.8, "raw_scale": 100, "source_record": "Qwen1.5-32B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-32B-Chat" }, { "id": "9a43355adba7fdd40f0b", "model": "AutoCoder-S-6.7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "AutoCoder-S-6.7B", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder_S_6.7B" }, { "id": "4afdad5104fa1b40e87e", "model": "Qwen1.5-72B-Chat", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "Qwen1.5-72B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-72B-Chat" }, { "id": "f2b2f5ff73e7fc7a96c4", "model": "StarCoder2-15B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "StarCoder2-15B", "source_date": "2024-02-29", "report_url": "https://huggingface.co/bigcode/starcoder2-15b" }, { "id": "789765d27879537523d7", "model": "Yi-1.5-9B-Chat", "organization": "01.AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.122, "raw_score": 12.2, "raw_scale": 100, "source_record": "Yi-1.5-9B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-9B-Chat" }, { "id": "7603f5344a4b9ae2ddec", "model": "DeepSeek-Coder-V2-Lite-Base", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.115, "raw_score": 11.5, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Lite-Base", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Base" }, { "id": "3fbe41841dfcda2d7566", "model": "CodeLlama-34B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.10800000000000001, "raw_score": 10.8, "raw_scale": 100, "source_record": "CodeLlama-34B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf" }, { "id": "825ef89de63469de2a9f", "model": "Granite-Code-20B-Base", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.10800000000000001, "raw_score": 10.8, "raw_scale": 100, "source_record": "Granite-Code-20B-Base", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-20b-code-base" }, { "id": "bb447031a6e7fe4c2360", "model": "Granite-Code-34B-Base", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.10800000000000001, "raw_score": 10.8, "raw_scale": 100, "source_record": "Granite-Code-34B-Base", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-34b-code-base" }, { "id": "0c8f593b88a57e3d6eb8", "model": "CodeGemma-7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.10099999999999999, "raw_score": 10.1, "raw_scale": 100, "source_record": "CodeGemma-7B", "source_date": "2024-04-10", "report_url": "https://huggingface.co/google/codegemma-7b" }, { "id": "3a33f63e651f14b9122a", "model": "Gemma-2-9B-Instruct", "organization": "Google", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.10099999999999999, "raw_score": 10.1, "raw_scale": 100, "source_record": "Gemma-2-9B-Instruct", "source_date": "2024-06-19", "report_url": "https://huggingface.co/google/gemma-2-9b-it" }, { "id": "98db673e5e6547a2035f", "model": "Granite-Code-20B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.10099999999999999, "raw_score": 10.1, "raw_scale": 100, "source_record": "Granite-Code-20B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-20b-code-instruct" }, { "id": "2a17dc1bb2909ce6fbfb", "model": "CodeLlama-13B-Base", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.095, "raw_score": 9.5, "raw_scale": 100, "source_record": "CodeLlama-13B-Base", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-13b-hf" }, { "id": "c886958321a7ab6bde93", "model": "Granite-3.0-8B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.095, "raw_score": 9.5, "raw_scale": 100, "source_record": "Granite-3.0-8B-Instruct", "source_date": "2024-10-21", "report_url": "https://huggingface.co/ibm-granite/granite-3.0-8b-instruct" }, { "id": "fb4a8330a8daf37f7970", "model": "Yi-1.5-34B", "organization": "01.AI", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.095, "raw_score": 9.5, "raw_scale": 100, "source_record": "Yi-1.5-34B", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-34B" }, { "id": "65ef3b93e8c26ddc5013", "model": "Yi-1.5-34B-Chat", "organization": "01.AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.095, "raw_score": 9.5, "raw_scale": 100, "source_record": "Yi-1.5-34B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-34B-Chat" }, { "id": "47ab37520afe432df077", "model": "OpenCoder-1.5B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "OpenCoder-1.5B-Instruct", "source_date": "2024-11-09", "report_url": "https://huggingface.co/infly/OpenCoder-1.5B-Instruct" }, { "id": "e61d28081296e928c7d9", "model": "ReflectionCoder-CL-7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "ReflectionCoder-CL-7B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-7B" }, { "id": "c5b8c76a405f96925998", "model": "StarCoder2-7B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "StarCoder2-7B", "source_date": "2024-02-29", "report_url": "https://huggingface.co/bigcode/starcoder2-7b" }, { "id": "ffb1921ef6794e8d0859", "model": "Yi-1.5-9B", "organization": "01.AI", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "Yi-1.5-9B", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-9B" }, { "id": "c7c8c0ebacd92289e5dd", "model": "Granite-Code-8B-Base", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.081, "raw_score": 8.1, "raw_scale": 100, "source_record": "Granite-Code-8B-Base", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-8b-code-base" }, { "id": "59f0b398b9efbb2f327e", "model": "Granite-Code-8B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.081, "raw_score": 8.1, "raw_scale": 100, "source_record": "Granite-Code-8B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-8b-code-instruct" }, { "id": "cce44d106c4b3990eee1", "model": "Llama-3-8B-Instruct", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.081, "raw_score": 8.1, "raw_scale": 100, "source_record": "Llama-3-8B-Instruct", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct" }, { "id": "c77667958a981959323b", "model": "CodeGemma-2B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.07400000000000001, "raw_score": 7.4, "raw_scale": 100, "source_record": "CodeGemma-2B", "source_date": "2024-04-10", "report_url": "https://huggingface.co/google/codegemma-2b" }, { "id": "74c47691997bd779916a", "model": "InternLM2.5-7B-Chat", "organization": "Shanghai AI Laboratory", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.07400000000000001, "raw_score": 7.4, "raw_scale": 100, "source_record": "InternLM2.5-7B-Chat", "source_date": "2024-07-03", "report_url": "https://huggingface.co/internlm/internlm2_5-7b-chat" }, { "id": "52f557045fc8516db57b", "model": "CodeLlama-13B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.068, "raw_score": 6.8, "raw_scale": 100, "source_record": "CodeLlama-13B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf" }, { "id": "1cf7cd4c4f5fe21df135", "model": "Yi-1.5-6B-Chat", "organization": "01.AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.068, "raw_score": 6.8, "raw_scale": 100, "source_record": "Yi-1.5-6B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-6B-Chat" }, { "id": "8c1dd3ea17320962ada3", "model": "DeepSeek-R1-Distill-Qwen-7B", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.061, "raw_score": 6.1, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-7B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" }, { "id": "41a1b9681ffdb82d9c4d", "model": "Llama-3-8B-Base", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.061, "raw_score": 6.1, "raw_scale": 100, "source_record": "Llama-3-8B-Base", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-8B" }, { "id": "f0039c47cca363813d23", "model": "Mistral-7B-Instruct-v0.3", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.061, "raw_score": 6.1, "raw_scale": 100, "source_record": "Mistral-7B-Instruct-v0.3", "source_date": "2024-05-22", "report_url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3" }, { "id": "6629c1aa64afd2993f3d", "model": "Qwen2.5-Coder-1.5B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.061, "raw_score": 6.1, "raw_scale": 100, "source_record": "Qwen2.5-Coder-1.5B-Instruct", "source_date": "2024-11-12", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct" }, { "id": "7a73af90e5907882b901", "model": "CodeLlama-7B-Base", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.054000000000000006, "raw_score": 5.4, "raw_scale": 100, "source_record": "CodeLlama-7B-Base", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-7b-hf" }, { "id": "b5a25695785d9f388c0e", "model": "Granite-Code-3B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.054000000000000006, "raw_score": 5.4, "raw_scale": 100, "source_record": "Granite-Code-3B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-3b-code-instruct" }, { "id": "cc9e2326b7c1594793c8", "model": "Llama-3.2-3B-Instruct", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.054000000000000006, "raw_score": 5.4, "raw_scale": 100, "source_record": "Llama-3.2-3B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct" }, { "id": "0565dc69d77f2ed642c5", "model": "Mistral-7B-v0.3", "organization": "Mistral AI", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.047, "raw_score": 4.7, "raw_scale": 100, "source_record": "Mistral-7B-v0.3", "source_date": "2024-05-22", "report_url": "https://huggingface.co/mistralai/Mistral-7B-v0.3" }, { "id": "54f0332f9b7255ed0edc", "model": "Qwen2.5-1.5B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.047, "raw_score": 4.7, "raw_scale": 100, "source_record": "Qwen2.5-1.5B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct" }, { "id": "4df7648b2a7ce499a014", "model": "StarCoder2-3B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.047, "raw_score": 4.7, "raw_scale": 100, "source_record": "StarCoder2-3B", "source_date": "2024-02-29", "report_url": "https://huggingface.co/bigcode/starcoder2-3b" }, { "id": "27a797d5d20833c33e68", "model": "CodeLlama-7B-Instruct", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.040999999999999995, "raw_score": 4.1, "raw_scale": 100, "source_record": "CodeLlama-7B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf" }, { "id": "8a19057d0a8f0bacaa03", "model": "Yi-1.5-6B", "organization": "01.AI", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.040999999999999995, "raw_score": 4.1, "raw_scale": 100, "source_record": "Yi-1.5-6B", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-6B" }, { "id": "65a1b6072c86b2f1a80a", "model": "DeepSeek-Coder-1.3B-Base", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.034, "raw_score": 3.4, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-Base", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base" }, { "id": "22fc60302c3cd4a2fd1f", "model": "DeepSeek-R1-Distill-Llama-8B", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.034, "raw_score": 3.4, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Llama-8B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B" }, { "id": "ec7337d7fc279611920e", "model": "OpenCodeInterpreter-DS-1.3B", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.034, "raw_score": 3.4, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-1.3B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-1.3B" }, { "id": "921aca304e1a6b4bcc32", "model": "DeepSeek-Coder-1.3B-Instruct", "organization": "DeepSeek", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.02, "raw_score": 2.0, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct" }, { "id": "2d399ef44ac721206ab9", "model": "Qwen2.5-0.5B-Instruct", "organization": "Alibaba", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.02, "raw_score": 2.0, "raw_scale": 100, "source_record": "Qwen2.5-0.5B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct" }, { "id": "5e7efb01085ccf1a0ff6", "model": "Granite-Code-3B-Base", "organization": "", "configuration": "Hard-Complete; prefill=True; prompted=False", "score": 0.013999999999999999, "raw_score": 1.4, "raw_scale": 100, "source_record": "Granite-Code-3B-Base", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-3b-code-base" }, { "id": "4a3bd480d04ad502597b", "model": "Llama-3.2-1B-Instruct", "organization": "Meta", "configuration": "Hard-Complete; prefill=True; prompted=True", "score": 0.013999999999999999, "raw_score": 1.4, "raw_scale": 100, "source_record": "Llama-3.2-1B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct" } ] }, { "id": "bigcodebench-full-instruct", "name": "BigCodeBench Full (Instruct)", "group": "编程", "unit": "%", "metric": "Calibrated pass@1", "protocol": { "en": "Full-Instruct, calibrated pass@1 with greedy decoding. Each run retains its prompt and prefilling settings.", "zh": "Full-Instruct，采用贪心解码的校准 pass@1。各条结果保留提示与预填设置。" }, "source": { "data_url": "https://bigcode-bench.github.io/results.json", "sha256": "2e24b0322c5b8f3059616cf8428300e1aa8e619bb1520bd09aa397067c66cf7c", "retrieved_at": "2026-11-02T05:59:24.757532+00:00", "name": "BigCodeBench", "url": "https://bigcode-bench.github.io/", "selector": "Full: pass@1.instruct; numeric scores including zero", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "79448501633ce100dfdc", "model": "GPT-4o-2024-05-13", "organization": "OpenAI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.511, "raw_score": 51.1, "raw_scale": 100, "source_record": "GPT-4o-2024-05-13", "source_date": "2024-05-13", "report_url": "https://openai.com/index/hello-gpt-4o/" }, { "id": "4973eaa8e3ba92b1a901", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.5, "raw_score": 50.0, "raw_scale": 100, "source_record": "DeepSeek-V3", "source_date": "2024-12-26", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3" }, { "id": "48b9f4b337c8e379a4d4", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.49700000000000005, "raw_score": 49.7, "raw_scale": 100, "source_record": "Llama-4-Maverick", "source_date": "2025-04-05", "report_url": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct" }, { "id": "3a5e1b691f54958de124", "model": "Quasar-Alpha", "organization": "", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.496, "raw_score": 49.6, "raw_scale": 100, "source_record": "Quasar-Alpha", "source_date": "2025-04-02", "report_url": "https://openrouter.ai/openrouter/quasar-alpha" }, { "id": "f0af06aac84ffa2ce581", "model": "Gemini-Exp-1114", "organization": "Google", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.49200000000000005, "raw_score": 49.2, "raw_scale": 100, "source_record": "Gemini-Exp-1114", "source_date": "2024-11-14", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "0a1f27c38a7a1134eb67", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.49, "raw_score": 49.0, "raw_scale": 100, "source_record": "Qwen2.5-Coder-32B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct" }, { "id": "28c8e7619670fca9aa09", "model": "DeepSeek-V2-Chat (2024-06-28)", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.489, "raw_score": 48.9, "raw_scale": 100, "source_record": "DeepSeek-V2-Chat (2024-06-28)", "source_date": "2024-06-28", "report_url": "https://www.deepseek.com/" }, { "id": "6ef3cfbbec8c41741b3a", "model": "GPT-4.1-Mini-2025-04-14", "organization": "OpenAI", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.489, "raw_score": 48.9, "raw_scale": 100, "source_record": "GPT-4.1-Mini-2025-04-14", "source_date": "2025-04-14", "report_url": "https://openai.com/index/gpt-4-1/" }, { "id": "09151d438cdc834ab3f3", "model": "DeepSeek-V2.5-1210", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.486, "raw_score": 48.6, "raw_scale": 100, "source_record": "DeepSeek-V2.5-1210", "source_date": "2024-12-10" }, { "id": "531cb4f68416b94fca45", "model": "DeepSeek-Coder-V2-Instruct", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.48200000000000004, "raw_score": 48.2, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct" }, { "id": "6c9f0d592b9ed2724c14", "model": "GPT-4-Turbo-2024-04-09", "organization": "OpenAI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.48200000000000004, "raw_score": 48.2, "raw_scale": 100, "source_record": "GPT-4-Turbo-2024-04-09", "source_date": "2024-04-09", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "c48068679d990e503b8f", "model": "Qwen2.5-Coder-14B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.48200000000000004, "raw_score": 48.2, "raw_scale": 100, "source_record": "Qwen2.5-Coder-14B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct" }, { "id": "1327a2256ab190d453a4", "model": "GPT-4o-2024-11-20", "organization": "OpenAI", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.48, "raw_score": 48.0, "raw_scale": 100, "source_record": "GPT-4o-2024-11-20", "source_date": "2024-11-20", "report_url": "https://openai.com/gpt-4o/" }, { "id": "737aa6ae38a6f6aca30a", "model": "Athene-V2-Chat", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.47200000000000003, "raw_score": 47.2, "raw_scale": 100, "source_record": "Athene-V2-Chat", "source_date": "2024-11-14", "report_url": "https://huggingface.co/Nexusflow/Athene-V2-Chat" }, { "id": "01ef0a20735eb6761f64", "model": "Gemini-Exp-1206", "organization": "Google", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.47, "raw_score": 47.0, "raw_scale": 100, "source_record": "Gemini-Exp-1206", "source_date": "2024-12-06", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "97d06434c085ed09497b", "model": "Llama-3.3-70B-Instruct", "organization": "Meta", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.469, "raw_score": 46.9, "raw_scale": 100, "source_record": "Llama-3.3-70B-Instruct", "source_date": "2024-12-19", "report_url": "https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct" }, { "id": "b662be56ed2b70e20dc0", "model": "Claude-3.5-Sonnet-20240620", "organization": "Anthropic", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.46799999999999997, "raw_score": 46.8, "raw_scale": 100, "source_record": "Claude-3.5-Sonnet-20240620", "source_date": "2024-06-20", "report_url": "https://claude.ai/" }, { "id": "47956ebc5472be0dba2b", "model": "Athene-V2-Agent", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.462, "raw_score": 46.2, "raw_scale": 100, "source_record": "Athene-V2-Agent", "source_date": "2024-11-14", "report_url": "https://huggingface.co/Nexusflow/Athene-V2-Agent" }, { "id": "a0a9e2fd09d658cbf6f3", "model": "Claude-3.5-Haiku-20241022", "organization": "Anthropic", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.461, "raw_score": 46.1, "raw_scale": 100, "source_record": "Claude-3.5-Haiku-20241022", "source_date": "2024-10-22", "report_url": "https://claude.ai/" }, { "id": "583c1909ceca9bf51b86", "model": "GPT-4o-mini-2024-07-18", "organization": "OpenAI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.461, "raw_score": 46.1, "raw_scale": 100, "source_record": "GPT-4o-mini-2024-07-18", "source_date": "2024-07-18", "report_url": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/" }, { "id": "55daabb935f5211c65ce", "model": "Llama-3.1-70B-Instruct", "organization": "Meta", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.461, "raw_score": 46.1, "raw_scale": 100, "source_record": "Llama-3.1-70B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct" }, { "id": "a208d7a5691d17f7397a", "model": "GPT-4-0613", "organization": "OpenAI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.46, "raw_score": 46.0, "raw_scale": 100, "source_record": "GPT-4-0613", "source_date": "2024-06-13", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "b9b4bad7f3f914d4700b", "model": "Gemini-2.0-Flash-Exp", "organization": "Google", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.45899999999999996, "raw_score": 45.9, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Exp", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "972a302e182b812fc2ec", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.45799999999999996, "raw_score": 45.8, "raw_scale": 100, "source_record": "Qwen2.5-72B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct" }, { "id": "aabe2216f3ad8fe8bf24", "model": "Hermes-2-Theta-Llama-3-70B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.456, "raw_score": 45.6, "raw_scale": 100, "source_record": "Hermes-2-Theta-Llama-3-70B", "source_date": "2024-06-24", "report_url": "https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-70B" }, { "id": "30b22f7eaedf70c07a8d", "model": "Claude-3-Opus-20240229", "organization": "Anthropic", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.455, "raw_score": 45.5, "raw_scale": 100, "source_record": "Claude-3-Opus-20240229", "source_date": "2024-02-29", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "1cec437ecaa9e7f35384", "model": "Phi-4", "organization": "Microsoft", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.455, "raw_score": 45.5, "raw_scale": 100, "source_record": "Phi-4", "source_date": "2024-12-13", "report_url": "https://huggingface.co/microsoft/phi-4" }, { "id": "35a07cb26cdf3628e3a4", "model": "Gemini-Exp-1121", "organization": "Google", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.45399999999999996, "raw_score": 45.4, "raw_scale": 100, "source_record": "Gemini-Exp-1121", "source_date": "2024-11-21", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "9bf57ffa8a433335d160", "model": "Mistral-Small-24B-Instruct-2501", "organization": "Mistral AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.45299999999999996, "raw_score": 45.3, "raw_scale": 100, "source_record": "Mistral-Small-24B-Instruct-2501", "source_date": "2025-01-31", "report_url": "https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501" }, { "id": "466c60dae478cf162b3d", "model": "Sky-T1-32B-Flash", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.451, "raw_score": 45.1, "raw_scale": 100, "source_record": "Sky-T1-32B-Flash", "source_date": "2025-01-12", "report_url": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Flash" }, { "id": "b9b6acf62918ca57f5d9", "model": "Qwen2.5-32B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.45, "raw_score": 45.0, "raw_scale": 100, "source_record": "Qwen2.5-32B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct" }, { "id": "4a32de71d2358d01473e", "model": "Sky-T1-32B-Preview", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.449, "raw_score": 44.9, "raw_scale": 100, "source_record": "Sky-T1-32B-Preview", "source_date": "2025-01-12", "report_url": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Preview" }, { "id": "0e9ec69535f16dabc818", "model": "Claude-3.5-Sonnet-20241022", "organization": "Anthropic", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.446, "raw_score": 44.6, "raw_scale": 100, "source_record": "Claude-3.5-Sonnet-20241022", "source_date": "2024-10-22", "report_url": "https://claude.ai/" }, { "id": "f0311033bb98d0757277", "model": "QwQ-32B-Preview", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.446, "raw_score": 44.6, "raw_scale": 100, "source_record": "QwQ-32B-Preview", "source_date": "2024-11-28", "report_url": "https://huggingface.co/Qwen/QwQ-32B-Preview" }, { "id": "0ba9009f78dccd01f469", "model": "DeepSeek-R1-Distill-Qwen-32B", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.439, "raw_score": 43.9, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-32B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" }, { "id": "a15e5ef8fcf830c14ff1", "model": "Gemini-1.5-Pro-API-0514", "organization": "Google", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.43799999999999994, "raw_score": 43.8, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-API-0514", "source_date": "2024-05-14", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "191e36d7c9e59ab973b8", "model": "Llama-3-70B-Instruct", "organization": "Meta", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.436, "raw_score": 43.6, "raw_scale": 100, "source_record": "Llama-3-70B-Instruct", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct" }, { "id": "a3bf0e562269986eeb52", "model": "Gemini-1.5-Flash-API-0514", "organization": "Google", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.435, "raw_score": 43.5, "raw_scale": 100, "source_record": "Gemini-1.5-Flash-API-0514", "source_date": "2024-05-14", "report_url": "https://deepmind.google/technologies/gemini/flash/" }, { "id": "691f84511d0ec15341c5", "model": "OpenCoder-8B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.43200000000000005, "raw_score": 43.2, "raw_scale": 100, "source_record": "OpenCoder-8B-Instruct", "source_date": "2024-11-09", "report_url": "https://huggingface.co/infly/OpenCoder-8B-Instruct" }, { "id": "31db15521e03453ab249", "model": "Gemma-2-27B-Instruct", "organization": "Google", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.428, "raw_score": 42.8, "raw_scale": 100, "source_record": "Gemma-2-27B-Instruct", "source_date": "2024-06-19", "report_url": "https://huggingface.co/google/gemma-2-27b-it" }, { "id": "41d099c2942e37049603", "model": "Llama-3-70B-Synthia-v3.5", "organization": "Meta", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.428, "raw_score": 42.8, "raw_scale": 100, "source_record": "Llama-3-70B-Synthia-v3.5", "source_date": "2024-05-27", "report_url": "https://huggingface.co/migtissera/Llama-3-70B-Synthia-v3.5" }, { "id": "88699baaa38a9adf9be3", "model": "Claude-3-Sonnet-20240229", "organization": "Anthropic", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.42700000000000005, "raw_score": 42.7, "raw_scale": 100, "source_record": "Claude-3-Sonnet-20240229", "source_date": "2024-02-29", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "94f34ece2cd608881244", "model": "ReflectionCoder-DS-33B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.424, "raw_score": 42.4, "raw_scale": 100, "source_record": "ReflectionCoder-DS-33B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-33B" }, { "id": "880826db1a9c61240b8d", "model": "DeepSeek-Coder-33B-Instruct", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.42, "raw_score": 42.0, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct" }, { "id": "1a2aca7e452afb3aeff3", "model": "Codestral-22B-v0.1", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.418, "raw_score": 41.8, "raw_scale": 100, "source_record": "Codestral-22B-v0.1", "source_date": "2024-05-23", "report_url": "https://huggingface.co/mistralai/Codestral-22B-v0.1" }, { "id": "2d906b4023f42a6866ad", "model": "WhiteRabbitNeo-33B-v1.5", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.41700000000000004, "raw_score": 41.7, "raw_scale": 100, "source_record": "WhiteRabbitNeo-33B-v1.5", "source_date": "2024-02-10", "report_url": "https://huggingface.co/WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5" }, { "id": "eb2e685465bc054d6df0", "model": "AutoCoder", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.40700000000000003, "raw_score": 40.7, "raw_scale": 100, "source_record": "AutoCoder", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder" }, { "id": "b8f265c234cd3f272e73", "model": "CodeLlama-70B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.40700000000000003, "raw_score": 40.7, "raw_scale": 100, "source_record": "CodeLlama-70B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf" }, { "id": "df56bf427cd5a73f58d8", "model": "Mixtral-8x22B-Instruct", "organization": "Mistral AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.406, "raw_score": 40.6, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct", "source_date": "2024-04-17", "report_url": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1" }, { "id": "0ffe809b06eb35a62f7d", "model": "DeepSeek-V2-Chat", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.40399999999999997, "raw_score": 40.4, "raw_scale": 100, "source_record": "DeepSeek-V2-Chat", "source_date": "2024-04-06", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat" }, { "id": "d01d5d37cd200a83a4d1", "model": "Qwen2.5-Coder-7B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.40399999999999997, "raw_score": 40.4, "raw_scale": 100, "source_record": "Qwen2.5-Coder-7B-Instruct", "source_date": "2024-11-12", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct" }, { "id": "457d7f85397735e8f881", "model": "CodeGeex4-All-9B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.4, "raw_score": 40.0, "raw_scale": 100, "source_record": "CodeGeex4-All-9B", "source_date": "2024-07-05", "report_url": "https://huggingface.co/THUDM/codegeex4-all-9b" }, { "id": "ed5a2c885a5a0e38e2b7", "model": "WarriorCoder-6.7B (Reproduced)", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.39899999999999997, "raw_score": 39.9, "raw_scale": 100, "source_record": "WarriorCoder-6.7B (Reproduced)", "source_date": "2025-02-18", "report_url": "https://arxiv.org/abs/2412.17395" }, { "id": "2039b67a32ea922cfcd5", "model": "Qwen2.5-14B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.39799999999999996, "raw_score": 39.8, "raw_scale": 100, "source_record": "Qwen2.5-14B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct" }, { "id": "db1c1e11f754025a1f92", "model": "CodeQwen1.5-7B-Chat", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.396, "raw_score": 39.6, "raw_scale": 100, "source_record": "CodeQwen1.5-7B-Chat", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat" }, { "id": "a06e8815c7b78da345b5", "model": "Nxcode-CQ-7B-Orpo", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.396, "raw_score": 39.6, "raw_scale": 100, "source_record": "Nxcode-CQ-7B-Orpo", "source_date": "2024-04-25", "report_url": "https://huggingface.co/NTQAI/Nxcode-CQ-7B-orpo" }, { "id": "82ee262512a54a309c96", "model": "Claude-3-Haiku-20240307", "organization": "Anthropic", "configuration": "Full-Instruct; prefill=False; prompted=True", "score": 0.39399999999999996, "raw_score": 39.4, "raw_scale": 100, "source_record": "Claude-3-Haiku-20240307", "source_date": "2024-03-07", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "7020a0f966a0ef1931fb", "model": "GPT-3.5-Turbo-0125", "organization": "OpenAI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.391, "raw_score": 39.1, "raw_scale": 100, "source_record": "GPT-3.5-Turbo-0125", "source_date": "2024-01-25", "report_url": "https://openai.com/index/new-embedding-models-and-api-updates" }, { "id": "1c5251e09c83d24fbea0", "model": "Llama-3.1-Nemotron-70B-Instruct", "organization": "Meta", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.387, "raw_score": 38.7, "raw_scale": 100, "source_record": "Llama-3.1-Nemotron-70B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF" }, { "id": "ce163386751f026ab2c1", "model": "Qwen2-72B-Chat", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.385, "raw_score": 38.5, "raw_scale": 100, "source_record": "Qwen2-72B-Chat", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-72B-Chat" }, { "id": "9fa936c7f7eef438953b", "model": "DeepCoder-14B-Preview", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.382, "raw_score": 38.2, "raw_scale": 100, "source_record": "DeepCoder-14B-Preview", "source_date": "2025-04-09", "report_url": "https://huggingface.co/agentica-org/DeepCoder-14B-Preview" }, { "id": "c98656b9280aa90ed5ab", "model": "Phind-CodeLlama-34B-v2", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.382, "raw_score": 38.2, "raw_scale": 100, "source_record": "Phind-CodeLlama-34B-v2", "source_date": "2023-08-25", "report_url": "https://huggingface.co/Phind/Phind-CodeLlama-34B-v2" }, { "id": "08683bbc96ba70cd61e3", "model": "DeepSeek-R1-Distill-Qwen-14B", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.381, "raw_score": 38.1, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-14B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B" }, { "id": "93c7acd979f4b385ae10", "model": "Yi-Coder-9B-Chat", "organization": "01.AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.381, "raw_score": 38.1, "raw_scale": 100, "source_record": "Yi-Coder-9B-Chat", "source_date": "2024-09-04", "report_url": "https://huggingface.co/01-ai/Yi-Coder-9B-Chat" }, { "id": "39651f4435d5050c9212", "model": "Artigenz-Coder-DS-6.7B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.38, "raw_score": 38.0, "raw_scale": 100, "source_record": "Artigenz-Coder-DS-6.7B", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Artigenz/Artigenz-Coder-DS-6.7B" }, { "id": "cd6f07fbcf06d98a1a81", "model": "ReflectionCoder-CL-34B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.377, "raw_score": 37.7, "raw_scale": 100, "source_record": "ReflectionCoder-CL-34B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-34B" }, { "id": "dbe1e7661453b52708e1", "model": "Yi-Large", "organization": "01.AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.377, "raw_score": 37.7, "raw_scale": 100, "source_record": "Yi-Large", "source_date": "2024-05-13", "report_url": "https://www.lingyiwanwu.com/" }, { "id": "21dadbca66e9a80cac7b", "model": "Phi-3-Medium-128K-Instruct", "organization": "Microsoft", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.376, "raw_score": 37.6, "raw_scale": 100, "source_record": "Phi-3-Medium-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-medium-128k-instruct" }, { "id": "b61019fb4a8a8670e461", "model": "Qwen2.5-7B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.376, "raw_score": 37.6, "raw_scale": 100, "source_record": "Qwen2.5-7B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct" }, { "id": "1d2fea03400612233f90", "model": "StarCoder2-15B-Instruct-v0.1", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.376, "raw_score": 37.6, "raw_scale": 100, "source_record": "StarCoder2-15B-Instruct-v0.1", "source_date": "2024-04-30", "report_url": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1" }, { "id": "f3313792cf5138dd4ea3", "model": "Hermes-2-Pro-Llama-3-70B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.37200000000000005, "raw_score": 37.2, "raw_scale": 100, "source_record": "Hermes-2-Pro-Llama-3-70B", "source_date": "2024-06-27", "report_url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-70B" }, { "id": "0424a91abbd6ada2af5b", "model": "C4AI-Command-R-08-2024", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.371, "raw_score": 37.1, "raw_scale": 100, "source_record": "C4AI-Command-R-08-2024", "source_date": "2024-08-30", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-08-2024" }, { "id": "fb1399a69666a126eaff", "model": "OpenCodeInterpreter-DS-6.7B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.371, "raw_score": 37.1, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-6.7B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B" }, { "id": "f78d9deb8a19887b291a", "model": "Tess-v2.5.2-Qwen2-72B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.37, "raw_score": 37.0, "raw_scale": 100, "source_record": "Tess-v2.5.2-Qwen2-72B", "source_date": "2024-07-18", "report_url": "https://huggingface.co/migtissera/Tess-v2.5.2-Qwen2-72B" }, { "id": "b452697db8b6e46ac420", "model": "Athene-70B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.368, "raw_score": 36.8, "raw_scale": 100, "source_record": "Athene-70B", "source_date": "2024-07-20", "report_url": "https://huggingface.co/Nexusflow/Athene-70B" }, { "id": "25841a38654c0d8cc1fa", "model": "DeepSeek-Coder-V2-Lite-Instruct", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.368, "raw_score": 36.8, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Lite-Instruct", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct" }, { "id": "3baa03ec31c5ad034912", "model": "Phi-3.1-Mini-128K-Instruct", "organization": "Microsoft", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.368, "raw_score": 36.8, "raw_scale": 100, "source_record": "Phi-3.1-Mini-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct" }, { "id": "e15aaa25c73230d63687", "model": "ReflectionCoder-DS-6.7B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.368, "raw_score": 36.8, "raw_scale": 100, "source_record": "ReflectionCoder-DS-6.7B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-6.7B" }, { "id": "d1ef871a823b03b69612", "model": "Magicoder-S-DS-6.7B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.36200000000000004, "raw_score": 36.2, "raw_scale": 100, "source_record": "Magicoder-S-DS-6.7B", "source_date": "2024-12-04", "report_url": "https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B" }, { "id": "9a43355adba7fdd40f0b", "model": "AutoCoder-S-6.7B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.361, "raw_score": 36.1, "raw_scale": 100, "source_record": "AutoCoder-S-6.7B", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder_S_6.7B" }, { "id": "b5bfc24428b007ff6475", "model": "Granite-Code-34B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.361, "raw_score": 36.1, "raw_scale": 100, "source_record": "Granite-Code-34B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-34b-code-instruct" }, { "id": "dd07a3eb0ca36bde1336", "model": "Mistral-Small-Instruct-2409", "organization": "Mistral AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.361, "raw_score": 36.1, "raw_scale": 100, "source_record": "Mistral-Small-Instruct-2409", "source_date": "2024-09-18", "report_url": "https://huggingface.co/mistralai/Mistral-Small-Instruct-2409" }, { "id": "80b8ad04ff5542d82114", "model": "Qwen2-57B-A14B", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.361, "raw_score": 36.1, "raw_scale": 100, "source_record": "Qwen2-57B-A14B", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-57B-A14B" }, { "id": "fc4539651aa3cda0fbdc", "model": "Codestral-Mamba", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.359, "raw_score": 35.9, "raw_scale": 100, "source_record": "Codestral-Mamba", "source_date": "2024-07-16", "report_url": "https://huggingface.co/mistralai/mamba-codestral-7B-v0.1" }, { "id": "584fc8797f1014af0fda", "model": "DeepSeek-Coder-6.7B-Instruct", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.355, "raw_score": 35.5, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct" }, { "id": "4a114af99c8d3b80e971", "model": "DeepSeek-R1-Distill-Llama-70B", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.353, "raw_score": 35.3, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Llama-70B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B" }, { "id": "3a8fc11881e03b27a72f", "model": "Qwen1.5-110B-Chat", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.35, "raw_score": 35.0, "raw_scale": 100, "source_record": "Qwen1.5-110B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-110B-Chat" }, { "id": "47ab37520afe432df077", "model": "OpenCoder-1.5B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.349, "raw_score": 34.9, "raw_scale": 100, "source_record": "OpenCoder-1.5B-Instruct", "source_date": "2024-11-09", "report_url": "https://huggingface.co/infly/OpenCoder-1.5B-Instruct" }, { "id": "3a33f63e651f14b9122a", "model": "Gemma-2-9B-Instruct", "organization": "Google", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.34700000000000003, "raw_score": 34.7, "raw_scale": 100, "source_record": "Gemma-2-9B-Instruct", "source_date": "2024-06-19", "report_url": "https://huggingface.co/google/gemma-2-9b-it" }, { "id": "789765d27879537523d7", "model": "Yi-1.5-9B-Chat", "organization": "01.AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.345, "raw_score": 34.5, "raw_scale": 100, "source_record": "Yi-1.5-9B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-9B-Chat" }, { "id": "98db673e5e6547a2035f", "model": "Granite-Code-20B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.34, "raw_score": 34.0, "raw_scale": 100, "source_record": "Granite-Code-20B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-20b-code-instruct" }, { "id": "172fc6a41d0f943afeb2", "model": "WaveCoder-Ultra-6.7B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.33899999999999997, "raw_score": 33.9, "raw_scale": 100, "source_record": "WaveCoder-Ultra-6.7B", "source_date": "2023-12-26", "report_url": "https://huggingface.co/microsoft/wavecoder-ultra-6.7b" }, { "id": "65ef3b93e8c26ddc5013", "model": "Yi-1.5-34B-Chat", "organization": "01.AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.33899999999999997, "raw_score": 33.9, "raw_scale": 100, "source_record": "Yi-1.5-34B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-34B-Chat" }, { "id": "505064d9efe3613d0dbb", "model": "Command R+", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.33799999999999997, "raw_score": 33.8, "raw_scale": 100, "source_record": "Command R+", "source_date": "2024-04-04", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-plus" }, { "id": "4afdad5104fa1b40e87e", "model": "Qwen1.5-72B-Chat", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.332, "raw_score": 33.2, "raw_scale": 100, "source_record": "Qwen1.5-72B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-72B-Chat" }, { "id": "f9b45bd93750ea1d1681", "model": "Llama-3.1-8B-Instruct", "organization": "Meta", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.32799999999999996, "raw_score": 32.8, "raw_scale": 100, "source_record": "Llama-3.1-8B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct" }, { "id": "d1dafd75f9b06dcff96e", "model": "Phi-3.5-Mini-Instruct", "organization": "Microsoft", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.32799999999999996, "raw_score": 32.8, "raw_scale": 100, "source_record": "Phi-3.5-Mini-Instruct", "source_date": "2024-08-21", "report_url": "https://huggingface.co/microsoft/Phi-3.5-mini-instruct" }, { "id": "18d08a65d81ce38b6d84", "model": "CodeGemma-7B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.32299999999999995, "raw_score": 32.3, "raw_scale": 100, "source_record": "CodeGemma-7B-Instruct", "source_date": "2024-04-10", "report_url": "https://huggingface.co/google/codegemma-7b-it" }, { "id": "d34c081bee7a70214e79", "model": "Qwen1.5-32B-Chat", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.32299999999999995, "raw_score": 32.3, "raw_scale": 100, "source_record": "Qwen1.5-32B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-32B-Chat" }, { "id": "f88cc9c0f66d0b1d4001", "model": "AutoCoder-QW-7B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.322, "raw_score": 32.2, "raw_scale": 100, "source_record": "AutoCoder-QW-7B", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder_QW_7B" }, { "id": "0667522f2464c8f617db", "model": "Mistral-Small-2402", "organization": "Mistral AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.321, "raw_score": 32.1, "raw_scale": 100, "source_record": "Mistral-Small-2402", "source_date": "2024-02-26", "report_url": "https://mistral.ai/news/mistral-large/" }, { "id": "cce44d106c4b3990eee1", "model": "Llama-3-8B-Instruct", "organization": "Meta", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.319, "raw_score": 31.9, "raw_scale": 100, "source_record": "Llama-3-8B-Instruct", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct" }, { "id": "12f23f959927b2011e48", "model": "Phi-3-Small-128K-Instruct", "organization": "Microsoft", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.311, "raw_score": 31.1, "raw_scale": 100, "source_record": "Phi-3-Small-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-small-128k-instruct" }, { "id": "3f74609b2e0f679e0594", "model": "Mistral-Large-2402", "organization": "Mistral AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.3, "raw_score": 30.0, "raw_scale": 100, "source_record": "Mistral-Large-2402", "source_date": "2024-02-26", "report_url": "https://mistral.ai/news/mistral-large/" }, { "id": "4a209c72393be25357eb", "model": "Phi-3-Mini-128K-Instruct", "organization": "Microsoft", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.29600000000000004, "raw_score": 29.6, "raw_scale": 100, "source_record": "Phi-3-Mini-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct" }, { "id": "c886958321a7ab6bde93", "model": "Granite-3.0-8B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.293, "raw_score": 29.3, "raw_scale": 100, "source_record": "Granite-3.0-8B-Instruct", "source_date": "2024-10-21", "report_url": "https://huggingface.co/ibm-granite/granite-3.0-8b-instruct" }, { "id": "e6e26b4b9e2582b38680", "model": "Qwen2-7B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.29100000000000004, "raw_score": 29.1, "raw_scale": 100, "source_record": "Qwen2-7B-Instruct", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-7B-Instruct" }, { "id": "3fbe41841dfcda2d7566", "model": "CodeLlama-34B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.29, "raw_score": 29.0, "raw_scale": 100, "source_record": "CodeLlama-34B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf" }, { "id": "52f557045fc8516db57b", "model": "CodeLlama-13B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.285, "raw_score": 28.5, "raw_scale": 100, "source_record": "CodeLlama-13B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf" }, { "id": "e61d28081296e928c7d9", "model": "ReflectionCoder-CL-7B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.284, "raw_score": 28.4, "raw_scale": 100, "source_record": "ReflectionCoder-CL-7B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-7B" }, { "id": "25647cdb7148cc992141", "model": "OpenChat-3.6-8B-20240522", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.281, "raw_score": 28.1, "raw_scale": 100, "source_record": "OpenChat-3.6-8B-20240522", "source_date": "2024-05-22", "report_url": "https://huggingface.co/openchat/openchat-3.6-8b-20240522" }, { "id": "6629c1aa64afd2993f3d", "model": "Qwen2.5-Coder-1.5B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.27, "raw_score": 27.0, "raw_scale": 100, "source_record": "Qwen2.5-Coder-1.5B-Instruct", "source_date": "2024-11-12", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct" }, { "id": "74c47691997bd779916a", "model": "InternLM2.5-7B-Chat", "organization": "Shanghai AI Laboratory", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.258, "raw_score": 25.8, "raw_scale": 100, "source_record": "InternLM2.5-7B-Chat", "source_date": "2024-07-03", "report_url": "https://huggingface.co/internlm/internlm2_5-7b-chat" }, { "id": "1cf7cd4c4f5fe21df135", "model": "Yi-1.5-6B-Chat", "organization": "01.AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.256, "raw_score": 25.6, "raw_scale": 100, "source_record": "Yi-1.5-6B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-6B-Chat" }, { "id": "ec7337d7fc279611920e", "model": "OpenCodeInterpreter-DS-1.3B", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.253, "raw_score": 25.3, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-1.3B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-1.3B" }, { "id": "cc9e2326b7c1594793c8", "model": "Llama-3.2-3B-Instruct", "organization": "Meta", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.23399999999999999, "raw_score": 23.4, "raw_scale": 100, "source_record": "Llama-3.2-3B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct" }, { "id": "921aca304e1a6b4bcc32", "model": "DeepSeek-Coder-1.3B-Instruct", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.228, "raw_score": 22.8, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct" }, { "id": "27a797d5d20833c33e68", "model": "CodeLlama-7B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.21899999999999997, "raw_score": 21.9, "raw_scale": 100, "source_record": "CodeLlama-7B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf" }, { "id": "d4bbc5246f09eeeed7bd", "model": "Granite-3.0-2B-Instruct", "organization": "", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.205, "raw_score": 20.5, "raw_scale": 100, "source_record": "Granite-3.0-2B-Instruct", "source_date": "2024-10-21", "report_url": "https://huggingface.co/ibm-granite/granite-3.0-2b-instruct" }, { "id": "54f0332f9b7255ed0edc", "model": "Qwen2.5-1.5B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.203, "raw_score": 20.3, "raw_scale": 100, "source_record": "Qwen2.5-1.5B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct" }, { "id": "f0039c47cca363813d23", "model": "Mistral-7B-Instruct-v0.3", "organization": "Mistral AI", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.195, "raw_score": 19.5, "raw_scale": 100, "source_record": "Mistral-7B-Instruct-v0.3", "source_date": "2024-05-22", "report_url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3" }, { "id": "8c1dd3ea17320962ada3", "model": "DeepSeek-R1-Distill-Qwen-7B", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.175, "raw_score": 17.5, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-7B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" }, { "id": "22fc60302c3cd4a2fd1f", "model": "DeepSeek-R1-Distill-Llama-8B", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.106, "raw_score": 10.6, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Llama-8B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B" }, { "id": "2d399ef44ac721206ab9", "model": "Qwen2.5-0.5B-Instruct", "organization": "Alibaba", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.08800000000000001, "raw_score": 8.8, "raw_scale": 100, "source_record": "Qwen2.5-0.5B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct" }, { "id": "4a3bd480d04ad502597b", "model": "Llama-3.2-1B-Instruct", "organization": "Meta", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.08199999999999999, "raw_score": 8.2, "raw_scale": 100, "source_record": "Llama-3.2-1B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct" }, { "id": "4da4589d256fc2bb8534", "model": "DeepSeek-R1-Distill-Qwen-1.5B", "organization": "DeepSeek", "configuration": "Full-Instruct; prefill=True; prompted=True", "score": 0.07, "raw_score": 7.0, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-1.5B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" } ] }, { "id": "bigcodebench-full-complete", "name": "BigCodeBench Full (Complete)", "group": "编程", "unit": "%", "metric": "Calibrated pass@1", "protocol": { "en": "Full-Complete, calibrated pass@1 with greedy decoding. Each run retains its prompt and prefilling settings.", "zh": "Full-Complete，采用贪心解码的校准 pass@1。各条结果保留提示与预填设置。" }, "source": { "data_url": "https://bigcode-bench.github.io/results.json", "sha256": "2e24b0322c5b8f3059616cf8428300e1aa8e619bb1520bd09aa397067c66cf7c", "retrieved_at": "2026-11-02T05:59:24.757532+00:00", "name": "BigCodeBench", "url": "https://bigcode-bench.github.io/", "selector": "Full: pass@1.complete; numeric scores including zero", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "01ef0a20735eb6761f64", "model": "Gemini-Exp-1206", "organization": "Google", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.624, "raw_score": 62.4, "raw_scale": 100, "source_record": "Gemini-Exp-1206", "source_date": "2024-12-06", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "4973eaa8e3ba92b1a901", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.622, "raw_score": 62.2, "raw_scale": 100, "source_record": "DeepSeek-V3", "source_date": "2024-12-26", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3" }, { "id": "48b9f4b337c8e379a4d4", "model": "Llama-4-Maverick", "organization": "Meta", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.614, "raw_score": 61.4, "raw_scale": 100, "source_record": "Llama-4-Maverick", "source_date": "2025-04-05", "report_url": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct" }, { "id": "79448501633ce100dfdc", "model": "GPT-4o-2024-05-13", "organization": "OpenAI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.611, "raw_score": 61.1, "raw_scale": 100, "source_record": "GPT-4o-2024-05-13", "source_date": "2024-05-13", "report_url": "https://openai.com/index/hello-gpt-4o/" }, { "id": "3a5e1b691f54958de124", "model": "Quasar-Alpha", "organization": "", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.606, "raw_score": 60.6, "raw_scale": 100, "source_record": "Quasar-Alpha", "source_date": "2025-04-02", "report_url": "https://openrouter.ai/openrouter/quasar-alpha" }, { "id": "b9b4bad7f3f914d4700b", "model": "Gemini-2.0-Flash-Exp", "organization": "Google", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.599, "raw_score": 59.9, "raw_scale": 100, "source_record": "Gemini-2.0-Flash-Exp", "source_date": "2025-02-05", "report_url": "https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/" }, { "id": "531cb4f68416b94fca45", "model": "DeepSeek-Coder-V2-Instruct", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.597, "raw_score": 59.7, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Instruct", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct" }, { "id": "28c8e7619670fca9aa09", "model": "DeepSeek-V2-Chat (2024-06-28)", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.594, "raw_score": 59.4, "raw_scale": 100, "source_record": "DeepSeek-V2-Chat (2024-06-28)", "source_date": "2024-06-28", "report_url": "https://www.deepseek.com/" }, { "id": "f0af06aac84ffa2ce581", "model": "Gemini-Exp-1114", "organization": "Google", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.593, "raw_score": 59.3, "raw_scale": 100, "source_record": "Gemini-Exp-1114", "source_date": "2024-11-14", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "a0a9e2fd09d658cbf6f3", "model": "Claude-3.5-Haiku-20241022", "organization": "Anthropic", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.59, "raw_score": 59.0, "raw_scale": 100, "source_record": "Claude-3.5-Haiku-20241022", "source_date": "2024-10-22", "report_url": "https://claude.ai/" }, { "id": "1327a2256ab190d453a4", "model": "GPT-4o-2024-11-20", "organization": "OpenAI", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.589, "raw_score": 58.9, "raw_scale": 100, "source_record": "GPT-4o-2024-11-20", "source_date": "2024-11-20", "report_url": "https://openai.com/gpt-4o/" }, { "id": "b662be56ed2b70e20dc0", "model": "Claude-3.5-Sonnet-20240620", "organization": "Anthropic", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.586, "raw_score": 58.6, "raw_scale": 100, "source_record": "Claude-3.5-Sonnet-20240620", "source_date": "2024-06-20", "report_url": "https://claude.ai/" }, { "id": "6c9f0d592b9ed2724c14", "model": "GPT-4-Turbo-2024-04-09", "organization": "OpenAI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.5820000000000001, "raw_score": 58.2, "raw_scale": 100, "source_record": "GPT-4-Turbo-2024-04-09", "source_date": "2024-04-09", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "35a07cb26cdf3628e3a4", "model": "Gemini-Exp-1121", "organization": "Google", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.581, "raw_score": 58.1, "raw_scale": 100, "source_record": "Gemini-Exp-1121", "source_date": "2024-11-21", "report_url": "https://deepmind.google/technologies/gemini" }, { "id": "0a1f27c38a7a1134eb67", "model": "Qwen2.5-Coder-32B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.58, "raw_score": 58.0, "raw_scale": 100, "source_record": "Qwen2.5-Coder-32B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct" }, { "id": "0e9ec69535f16dabc818", "model": "Claude-3.5-Sonnet-20241022", "organization": "Anthropic", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.575, "raw_score": 57.5, "raw_scale": 100, "source_record": "Claude-3.5-Sonnet-20241022", "source_date": "2024-10-22", "report_url": "https://claude.ai/" }, { "id": "a15e5ef8fcf830c14ff1", "model": "Gemini-1.5-Pro-API-0514", "organization": "Google", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.575, "raw_score": 57.5, "raw_scale": 100, "source_record": "Gemini-1.5-Pro-API-0514", "source_date": "2024-05-14", "report_url": "https://deepmind.google/technologies/gemini/pro" }, { "id": "97d06434c085ed09497b", "model": "Llama-3.3-70B-Instruct", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.575, "raw_score": 57.5, "raw_scale": 100, "source_record": "Llama-3.3-70B-Instruct", "source_date": "2024-12-19", "report_url": "https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct" }, { "id": "30b22f7eaedf70c07a8d", "model": "Claude-3-Opus-20240229", "organization": "Anthropic", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.574, "raw_score": 57.4, "raw_scale": 100, "source_record": "Claude-3-Opus-20240229", "source_date": "2024-02-29", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "583c1909ceca9bf51b86", "model": "GPT-4o-mini-2024-07-18", "organization": "OpenAI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.574, "raw_score": 57.4, "raw_scale": 100, "source_record": "GPT-4o-mini-2024-07-18", "source_date": "2024-07-18", "report_url": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/" }, { "id": "a208d7a5691d17f7397a", "model": "GPT-4-0613", "organization": "OpenAI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.5720000000000001, "raw_score": 57.2, "raw_scale": 100, "source_record": "GPT-4-0613", "source_date": "2024-06-13", "report_url": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday" }, { "id": "737aa6ae38a6f6aca30a", "model": "Athene-V2-Chat", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.568, "raw_score": 56.8, "raw_scale": 100, "source_record": "Athene-V2-Chat", "source_date": "2024-11-14", "report_url": "https://huggingface.co/Nexusflow/Athene-V2-Chat" }, { "id": "c48068679d990e503b8f", "model": "Qwen2.5-Coder-14B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.5670000000000001, "raw_score": 56.7, "raw_scale": 100, "source_record": "Qwen2.5-Coder-14B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct" }, { "id": "47956ebc5472be0dba2b", "model": "Athene-V2-Agent", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.561, "raw_score": 56.1, "raw_scale": 100, "source_record": "Athene-V2-Agent", "source_date": "2024-11-14", "report_url": "https://huggingface.co/Nexusflow/Athene-V2-Agent" }, { "id": "972a302e182b812fc2ec", "model": "Qwen2.5-72B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.5589999999999999, "raw_score": 55.9, "raw_scale": 100, "source_record": "Qwen2.5-72B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct" }, { "id": "aabe2216f3ad8fe8bf24", "model": "Hermes-2-Theta-Llama-3-70B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.556, "raw_score": 55.6, "raw_scale": 100, "source_record": "Hermes-2-Theta-Llama-3-70B", "source_date": "2024-06-24", "report_url": "https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-70B" }, { "id": "1cec437ecaa9e7f35384", "model": "Phi-4", "organization": "Microsoft", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.5539999999999999, "raw_score": 55.4, "raw_scale": 100, "source_record": "Phi-4", "source_date": "2024-12-13", "report_url": "https://huggingface.co/microsoft/phi-4" }, { "id": "a3bf0e562269986eeb52", "model": "Gemini-1.5-Flash-API-0514", "organization": "Google", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.551, "raw_score": 55.1, "raw_scale": 100, "source_record": "Gemini-1.5-Flash-API-0514", "source_date": "2024-05-14", "report_url": "https://deepmind.google/technologies/gemini/flash/" }, { "id": "0ba9009f78dccd01f469", "model": "DeepSeek-R1-Distill-Qwen-32B", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.5489999999999999, "raw_score": 54.9, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-32B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" }, { "id": "55daabb935f5211c65ce", "model": "Llama-3.1-70B-Instruct", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.5479999999999999, "raw_score": 54.8, "raw_scale": 100, "source_record": "Llama-3.1-70B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct" }, { "id": "f3313792cf5138dd4ea3", "model": "Hermes-2-Pro-Llama-3-70B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.545, "raw_score": 54.5, "raw_scale": 100, "source_record": "Hermes-2-Pro-Llama-3-70B", "source_date": "2024-06-27", "report_url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-70B" }, { "id": "191e36d7c9e59ab973b8", "model": "Llama-3-70B-Instruct", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.545, "raw_score": 54.5, "raw_scale": 100, "source_record": "Llama-3-70B-Instruct", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct" }, { "id": "f0311033bb98d0757277", "model": "QwQ-32B-Preview", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.544, "raw_score": 54.4, "raw_scale": 100, "source_record": "QwQ-32B-Preview", "source_date": "2024-11-28", "report_url": "https://huggingface.co/Qwen/QwQ-32B-Preview" }, { "id": "41d099c2942e37049603", "model": "Llama-3-70B-Synthia-v3.5", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.541, "raw_score": 54.1, "raw_scale": 100, "source_record": "Llama-3-70B-Synthia-v3.5", "source_date": "2024-05-27", "report_url": "https://huggingface.co/migtissera/Llama-3-70B-Synthia-v3.5" }, { "id": "ce163386751f026ab2c1", "model": "Qwen2-72B-Chat", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.54, "raw_score": 54.0, "raw_scale": 100, "source_record": "Qwen2-72B-Chat", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-72B-Chat" }, { "id": "88699baaa38a9adf9be3", "model": "Claude-3-Sonnet-20240229", "organization": "Anthropic", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.5379999999999999, "raw_score": 53.8, "raw_scale": 100, "source_record": "Claude-3-Sonnet-20240229", "source_date": "2024-02-29", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "09151d438cdc834ab3f3", "model": "DeepSeek-V2.5-1210", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.532, "raw_score": 53.2, "raw_scale": 100, "source_record": "DeepSeek-V2.5-1210", "source_date": "2024-12-10" }, { "id": "94f34ece2cd608881244", "model": "ReflectionCoder-DS-33B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.529, "raw_score": 52.9, "raw_scale": 100, "source_record": "ReflectionCoder-DS-33B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-33B" }, { "id": "2d906b4023f42a6866ad", "model": "WhiteRabbitNeo-33B-v1.5", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.526, "raw_score": 52.6, "raw_scale": 100, "source_record": "WhiteRabbitNeo-33B-v1.5", "source_date": "2024-02-10", "report_url": "https://huggingface.co/WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5" }, { "id": "eb2e685465bc054d6df0", "model": "AutoCoder", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.525, "raw_score": 52.5, "raw_scale": 100, "source_record": "AutoCoder", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder" }, { "id": "1a2aca7e452afb3aeff3", "model": "Codestral-22B-v0.1", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.525, "raw_score": 52.5, "raw_scale": 100, "source_record": "Codestral-22B-v0.1", "source_date": "2024-05-23", "report_url": "https://huggingface.co/mistralai/Codestral-22B-v0.1" }, { "id": "31db15521e03453ab249", "model": "Gemma-2-27B-Instruct", "organization": "Google", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.525, "raw_score": 52.5, "raw_scale": 100, "source_record": "Gemma-2-27B-Instruct", "source_date": "2024-06-19", "report_url": "https://huggingface.co/google/gemma-2-27b-it" }, { "id": "b9b6acf62918ca57f5d9", "model": "Qwen2.5-32B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.523, "raw_score": 52.3, "raw_scale": 100, "source_record": "Qwen2.5-32B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct" }, { "id": "2039b67a32ea922cfcd5", "model": "Qwen2.5-14B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.522, "raw_score": 52.2, "raw_scale": 100, "source_record": "Qwen2.5-14B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct" }, { "id": "466c60dae478cf162b3d", "model": "Sky-T1-32B-Flash", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.512, "raw_score": 51.2, "raw_scale": 100, "source_record": "Sky-T1-32B-Flash", "source_date": "2025-01-12", "report_url": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Flash" }, { "id": "880826db1a9c61240b8d", "model": "DeepSeek-Coder-33B-Instruct", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.511, "raw_score": 51.1, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct" }, { "id": "4a32de71d2358d01473e", "model": "Sky-T1-32B-Preview", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.511, "raw_score": 51.1, "raw_scale": 100, "source_record": "Sky-T1-32B-Preview", "source_date": "2025-01-12", "report_url": "https://huggingface.co/NovaSky-AI/Sky-T1-32B-Preview" }, { "id": "56928ca9ba87a6db3cc6", "model": "OpenCodeInterpreter-DS-33B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.51, "raw_score": 51.0, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-33B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B" }, { "id": "691f84511d0ec15341c5", "model": "OpenCoder-8B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.509, "raw_score": 50.9, "raw_scale": 100, "source_record": "OpenCoder-8B-Instruct", "source_date": "2024-11-09", "report_url": "https://huggingface.co/infly/OpenCoder-8B-Instruct" }, { "id": "7020a0f966a0ef1931fb", "model": "GPT-3.5-Turbo-0125", "organization": "OpenAI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.506, "raw_score": 50.6, "raw_scale": 100, "source_record": "GPT-3.5-Turbo-0125", "source_date": "2024-01-25", "report_url": "https://openai.com/index/new-embedding-models-and-api-updates" }, { "id": "9bf57ffa8a433335d160", "model": "Mistral-Small-24B-Instruct-2501", "organization": "Mistral AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.504, "raw_score": 50.4, "raw_scale": 100, "source_record": "Mistral-Small-24B-Instruct-2501", "source_date": "2025-01-31", "report_url": "https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501" }, { "id": "df56bf427cd5a73f58d8", "model": "Mixtral-8x22B-Instruct", "organization": "Mistral AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.502, "raw_score": 50.2, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct", "source_date": "2024-04-17", "report_url": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1" }, { "id": "82ee262512a54a309c96", "model": "Claude-3-Haiku-20240307", "organization": "Anthropic", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.501, "raw_score": 50.1, "raw_scale": 100, "source_record": "Claude-3-Haiku-20240307", "source_date": "2024-03-07", "report_url": "https://www.anthropic.com/news/claude-3-family" }, { "id": "4a114af99c8d3b80e971", "model": "DeepSeek-R1-Distill-Llama-70B", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.499, "raw_score": 49.9, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Llama-70B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B" }, { "id": "78dd22077aaf3bb7c874", "model": "KwaiCoder-23B-A4B-v1", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.499, "raw_score": 49.9, "raw_scale": 100, "source_record": "KwaiCoder-23B-A4B-v1", "source_date": "2025-01-25", "report_url": "https://huggingface.co/Kwaipilot/KwaiCoder-23B-A4B-v1" }, { "id": "b8f265c234cd3f272e73", "model": "CodeLlama-70B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.496, "raw_score": 49.6, "raw_scale": 100, "source_record": "CodeLlama-70B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf" }, { "id": "9fa936c7f7eef438953b", "model": "DeepCoder-14B-Preview", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.496, "raw_score": 49.6, "raw_scale": 100, "source_record": "DeepCoder-14B-Preview", "source_date": "2025-04-09", "report_url": "https://huggingface.co/agentica-org/DeepCoder-14B-Preview" }, { "id": "457d7f85397735e8f881", "model": "CodeGeex4-All-9B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.49, "raw_score": 49.0, "raw_scale": 100, "source_record": "CodeGeex4-All-9B", "source_date": "2024-07-05", "report_url": "https://huggingface.co/THUDM/codegeex4-all-9b" }, { "id": "0ffe809b06eb35a62f7d", "model": "DeepSeek-V2-Chat", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.49, "raw_score": 49.0, "raw_scale": 100, "source_record": "DeepSeek-V2-Chat", "source_date": "2024-04-06", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat" }, { "id": "93c7acd979f4b385ae10", "model": "Yi-Coder-9B-Chat", "organization": "01.AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.49, "raw_score": 49.0, "raw_scale": 100, "source_record": "Yi-Coder-9B-Chat", "source_date": "2024-09-04", "report_url": "https://huggingface.co/01-ai/Yi-Coder-9B-Chat" }, { "id": "d01d5d37cd200a83a4d1", "model": "Qwen2.5-Coder-7B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.488, "raw_score": 48.8, "raw_scale": 100, "source_record": "Qwen2.5-Coder-7B-Instruct", "source_date": "2024-11-12", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct" }, { "id": "21dadbca66e9a80cac7b", "model": "Phi-3-Medium-128K-Instruct", "organization": "Microsoft", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.48700000000000004, "raw_score": 48.7, "raw_scale": 100, "source_record": "Phi-3-Medium-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-medium-128k-instruct" }, { "id": "08683bbc96ba70cd61e3", "model": "DeepSeek-R1-Distill-Qwen-14B", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.484, "raw_score": 48.4, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-14B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B" }, { "id": "1c5251e09c83d24fbea0", "model": "Llama-3.1-Nemotron-70B-Instruct", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.48200000000000004, "raw_score": 48.2, "raw_scale": 100, "source_record": "Llama-3.1-Nemotron-70B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF" }, { "id": "e15aaa25c73230d63687", "model": "ReflectionCoder-DS-6.7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.478, "raw_score": 47.8, "raw_scale": 100, "source_record": "ReflectionCoder-DS-6.7B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-DS-6.7B" }, { "id": "25841a38654c0d8cc1fa", "model": "DeepSeek-Coder-V2-Lite-Instruct", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.47600000000000003, "raw_score": 47.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Lite-Instruct", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct" }, { "id": "d1ef871a823b03b69612", "model": "Magicoder-S-DS-6.7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.47600000000000003, "raw_score": 47.6, "raw_scale": 100, "source_record": "Magicoder-S-DS-6.7B", "source_date": "2024-12-04", "report_url": "https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B" }, { "id": "b452697db8b6e46ac420", "model": "Athene-70B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.475, "raw_score": 47.5, "raw_scale": 100, "source_record": "Athene-70B", "source_date": "2024-07-20", "report_url": "https://huggingface.co/Nexusflow/Athene-70B" }, { "id": "c98656b9280aa90ed5ab", "model": "Phind-CodeLlama-34B-v2", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.475, "raw_score": 47.5, "raw_scale": 100, "source_record": "Phind-CodeLlama-34B-v2", "source_date": "2023-08-25", "report_url": "https://huggingface.co/Phind/Phind-CodeLlama-34B-v2" }, { "id": "dbe1e7661453b52708e1", "model": "Yi-Large", "organization": "01.AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.47200000000000003, "raw_score": 47.2, "raw_scale": 100, "source_record": "Yi-Large", "source_date": "2024-05-13", "report_url": "https://www.lingyiwanwu.com/" }, { "id": "39651f4435d5050c9212", "model": "Artigenz-Coder-DS-6.7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.47100000000000003, "raw_score": 47.1, "raw_scale": 100, "source_record": "Artigenz-Coder-DS-6.7B", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Artigenz/Artigenz-Coder-DS-6.7B" }, { "id": "80b8ad04ff5542d82114", "model": "Qwen2-57B-A14B", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.46799999999999997, "raw_score": 46.8, "raw_scale": 100, "source_record": "Qwen2-57B-A14B", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-57B-A14B" }, { "id": "cd6f07fbcf06d98a1a81", "model": "ReflectionCoder-CL-34B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.46799999999999997, "raw_score": 46.8, "raw_scale": 100, "source_record": "ReflectionCoder-CL-34B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-34B" }, { "id": "f3dde70fbdce9fea442f", "model": "DeepSeek-Coder-33B-Base", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.466, "raw_score": 46.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-33B-Base", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-base" }, { "id": "dd07a3eb0ca36bde1336", "model": "Mistral-Small-Instruct-2409", "organization": "Mistral AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.466, "raw_score": 46.6, "raw_scale": 100, "source_record": "Mistral-Small-Instruct-2409", "source_date": "2024-09-18", "report_url": "https://huggingface.co/mistralai/Mistral-Small-Instruct-2409" }, { "id": "9a43355adba7fdd40f0b", "model": "AutoCoder-S-6.7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.465, "raw_score": 46.5, "raw_scale": 100, "source_record": "AutoCoder-S-6.7B", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder_S_6.7B" }, { "id": "f78d9deb8a19887b291a", "model": "Tess-v2.5.2-Qwen2-72B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.46299999999999997, "raw_score": 46.3, "raw_scale": 100, "source_record": "Tess-v2.5.2-Qwen2-72B", "source_date": "2024-07-18", "report_url": "https://huggingface.co/migtissera/Tess-v2.5.2-Qwen2-72B" }, { "id": "3baa03ec31c5ad034912", "model": "Phi-3.1-Mini-128K-Instruct", "organization": "Microsoft", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.462, "raw_score": 46.2, "raw_scale": 100, "source_record": "Phi-3.1-Mini-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct" }, { "id": "b61019fb4a8a8670e461", "model": "Qwen2.5-7B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.461, "raw_score": 46.1, "raw_scale": 100, "source_record": "Qwen2.5-7B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct" }, { "id": "bb051df20f0935ee2e27", "model": "CodeQwen1.5-7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.456, "raw_score": 45.6, "raw_scale": 100, "source_record": "CodeQwen1.5-7B", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Qwen/CodeQwen1.5-7B" }, { "id": "e532aaf956fe2b5398af", "model": "Mixtral-8x22B-Base", "organization": "Mistral AI", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.45299999999999996, "raw_score": 45.3, "raw_scale": 100, "source_record": "Mixtral-8x22B-Base", "source_date": "2024-04-17", "report_url": "https://huggingface.co/mistralai/Mixtral-8x22B-v0.1" }, { "id": "0424a91abbd6ada2af5b", "model": "C4AI-Command-R-08-2024", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.452, "raw_score": 45.2, "raw_scale": 100, "source_record": "C4AI-Command-R-08-2024", "source_date": "2024-08-30", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-08-2024" }, { "id": "1d2fea03400612233f90", "model": "StarCoder2-15B-Instruct-v0.1", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.451, "raw_score": 45.1, "raw_scale": 100, "source_record": "StarCoder2-15B-Instruct-v0.1", "source_date": "2024-04-30", "report_url": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1" }, { "id": "fc4539651aa3cda0fbdc", "model": "Codestral-Mamba", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.446, "raw_score": 44.6, "raw_scale": 100, "source_record": "Codestral-Mamba", "source_date": "2024-07-16", "report_url": "https://huggingface.co/mistralai/mamba-codestral-7B-v0.1" }, { "id": "a06e8815c7b78da345b5", "model": "Nxcode-CQ-7B-Orpo", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.446, "raw_score": 44.6, "raw_scale": 100, "source_record": "Nxcode-CQ-7B-Orpo", "source_date": "2024-04-25", "report_url": "https://huggingface.co/NTQAI/Nxcode-CQ-7B-orpo" }, { "id": "fb1399a69666a126eaff", "model": "OpenCodeInterpreter-DS-6.7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.446, "raw_score": 44.6, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-6.7B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B" }, { "id": "b5bfc24428b007ff6475", "model": "Granite-Code-34B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.444, "raw_score": 44.4, "raw_scale": 100, "source_record": "Granite-Code-34B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-34b-code-instruct" }, { "id": "3a8fc11881e03b27a72f", "model": "Qwen1.5-110B-Chat", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.444, "raw_score": 44.4, "raw_scale": 100, "source_record": "Qwen1.5-110B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-110B-Chat" }, { "id": "50071bd2dc1398fc8866", "model": "CodeLlama-70B-Base", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.44, "raw_score": 44.0, "raw_scale": 100, "source_record": "CodeLlama-70B-Base", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-70b-hf" }, { "id": "584fc8797f1014af0fda", "model": "DeepSeek-Coder-6.7B-Instruct", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.43799999999999994, "raw_score": 43.8, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct" }, { "id": "65ef3b93e8c26ddc5013", "model": "Yi-1.5-34B-Chat", "organization": "01.AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.43799999999999994, "raw_score": 43.8, "raw_scale": 100, "source_record": "Yi-1.5-34B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-34B-Chat" }, { "id": "172fc6a41d0f943afeb2", "model": "WaveCoder-Ultra-6.7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.43700000000000006, "raw_score": 43.7, "raw_scale": 100, "source_record": "WaveCoder-Ultra-6.7B", "source_date": "2023-12-26", "report_url": "https://huggingface.co/microsoft/wavecoder-ultra-6.7b" }, { "id": "db1c1e11f754025a1f92", "model": "CodeQwen1.5-7B-Chat", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.436, "raw_score": 43.6, "raw_scale": 100, "source_record": "CodeQwen1.5-7B-Chat", "source_date": "2024-04-16", "report_url": "https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat" }, { "id": "883f1f2bbff22c83d490", "model": "Llama-3-70B-Base", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.433, "raw_score": 43.3, "raw_scale": 100, "source_record": "Llama-3-70B-Base", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-70B" }, { "id": "fb257208509ac862fac3", "model": "Llama-4-Scout", "organization": "Meta", "configuration": "Full-Complete; prefill=False; prompted=True", "score": 0.431, "raw_score": 43.1, "raw_scale": 100, "source_record": "Llama-4-Scout", "source_date": "2025-04-05", "report_url": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct" }, { "id": "789765d27879537523d7", "model": "Yi-1.5-9B-Chat", "organization": "01.AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.424, "raw_score": 42.4, "raw_scale": 100, "source_record": "Yi-1.5-9B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-9B-Chat" }, { "id": "e6e26b4b9e2582b38680", "model": "Qwen2-7B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "Qwen2-7B-Instruct", "source_date": "2024-06-07", "report_url": "https://huggingface.co/Qwen/Qwen2-7B-Instruct" }, { "id": "98db673e5e6547a2035f", "model": "Granite-Code-20B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.42, "raw_score": 42.0, "raw_scale": 100, "source_record": "Granite-Code-20B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-20b-code-instruct" }, { "id": "d34c081bee7a70214e79", "model": "Qwen1.5-32B-Chat", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.42, "raw_score": 42.0, "raw_scale": 100, "source_record": "Qwen1.5-32B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-32B-Chat" }, { "id": "505064d9efe3613d0dbb", "model": "Command R+", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.419, "raw_score": 41.9, "raw_scale": 100, "source_record": "Command R+", "source_date": "2024-04-04", "report_url": "https://huggingface.co/CohereForAI/c4ai-command-r-plus" }, { "id": "f88cc9c0f66d0b1d4001", "model": "AutoCoder-QW-7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.418, "raw_score": 41.8, "raw_scale": 100, "source_record": "AutoCoder-QW-7B", "source_date": "2024-05-23", "report_url": "https://huggingface.co/Bin12345/AutoCoder_QW_7B" }, { "id": "10fc70e5dc28194ef284", "model": "DeepSeek-Coder-6.7B-Base", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.418, "raw_score": 41.8, "raw_scale": 100, "source_record": "DeepSeek-Coder-6.7B-Base", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base" }, { "id": "0667522f2464c8f617db", "model": "Mistral-Small-2402", "organization": "Mistral AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.413, "raw_score": 41.3, "raw_scale": 100, "source_record": "Mistral-Small-2402", "source_date": "2024-02-26", "report_url": "https://mistral.ai/news/mistral-large/" }, { "id": "3a33f63e651f14b9122a", "model": "Gemma-2-9B-Instruct", "organization": "Google", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.406, "raw_score": 40.6, "raw_scale": 100, "source_record": "Gemma-2-9B-Instruct", "source_date": "2024-06-19", "report_url": "https://huggingface.co/google/gemma-2-9b-it" }, { "id": "4a209c72393be25357eb", "model": "Phi-3-Mini-128K-Instruct", "organization": "Microsoft", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.406, "raw_score": 40.6, "raw_scale": 100, "source_record": "Phi-3-Mini-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct" }, { "id": "f9b45bd93750ea1d1681", "model": "Llama-3.1-8B-Instruct", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "Llama-3.1-8B-Instruct", "source_date": "2024-07-23", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct" }, { "id": "e61d28081296e928c7d9", "model": "ReflectionCoder-CL-7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "ReflectionCoder-CL-7B", "source_date": "2024-05-27", "report_url": "https://huggingface.co/SenseLLM/ReflectionCoder-CL-7B" }, { "id": "4afdad5104fa1b40e87e", "model": "Qwen1.5-72B-Chat", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.40299999999999997, "raw_score": 40.3, "raw_scale": 100, "source_record": "Qwen1.5-72B-Chat", "source_date": "2024-04-26", "report_url": "https://huggingface.co/Qwen/Qwen1.5-72B-Chat" }, { "id": "fb4a8330a8daf37f7970", "model": "Yi-1.5-34B", "organization": "01.AI", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.4, "raw_score": 40.0, "raw_scale": 100, "source_record": "Yi-1.5-34B", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-34B" }, { "id": "59f0b398b9efbb2f327e", "model": "Granite-Code-8B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.397, "raw_score": 39.7, "raw_scale": 100, "source_record": "Granite-Code-8B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-8b-code-instruct" }, { "id": "12f23f959927b2011e48", "model": "Phi-3-Small-128K-Instruct", "organization": "Microsoft", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.39399999999999996, "raw_score": 39.4, "raw_scale": 100, "source_record": "Phi-3-Small-128K-Instruct", "source_date": "2024-05-21", "report_url": "https://huggingface.co/microsoft/Phi-3-small-128k-instruct" }, { "id": "18d08a65d81ce38b6d84", "model": "CodeGemma-7B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.39299999999999996, "raw_score": 39.3, "raw_scale": 100, "source_record": "CodeGemma-7B-Instruct", "source_date": "2024-04-10", "report_url": "https://huggingface.co/google/codegemma-7b-it" }, { "id": "47ab37520afe432df077", "model": "OpenCoder-1.5B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.38799999999999996, "raw_score": 38.8, "raw_scale": 100, "source_record": "OpenCoder-1.5B-Instruct", "source_date": "2024-11-09", "report_url": "https://huggingface.co/infly/OpenCoder-1.5B-Instruct" }, { "id": "d1dafd75f9b06dcff96e", "model": "Phi-3.5-Mini-Instruct", "organization": "Microsoft", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.385, "raw_score": 38.5, "raw_scale": 100, "source_record": "Phi-3.5-Mini-Instruct", "source_date": "2024-08-21", "report_url": "https://huggingface.co/microsoft/Phi-3.5-mini-instruct" }, { "id": "bb447031a6e7fe4c2360", "model": "Granite-Code-34B-Base", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.384, "raw_score": 38.4, "raw_scale": 100, "source_record": "Granite-Code-34B-Base", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-34b-code-base" }, { "id": "f2b2f5ff73e7fc7a96c4", "model": "StarCoder2-15B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.384, "raw_score": 38.4, "raw_scale": 100, "source_record": "StarCoder2-15B", "source_date": "2024-02-29", "report_url": "https://huggingface.co/bigcode/starcoder2-15b" }, { "id": "0c8f593b88a57e3d6eb8", "model": "CodeGemma-7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.38299999999999995, "raw_score": 38.3, "raw_scale": 100, "source_record": "CodeGemma-7B", "source_date": "2024-04-10", "report_url": "https://huggingface.co/google/codegemma-7b" }, { "id": "3f74609b2e0f679e0594", "model": "Mistral-Large-2402", "organization": "Mistral AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.38299999999999995, "raw_score": 38.3, "raw_scale": 100, "source_record": "Mistral-Large-2402", "source_date": "2024-02-26", "report_url": "https://mistral.ai/news/mistral-large/" }, { "id": "cfebb5ea5cad76711273", "model": "CodeLlama-34B-Base", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.371, "raw_score": 37.1, "raw_scale": 100, "source_record": "CodeLlama-34B-Base", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-34b-hf" }, { "id": "25647cdb7148cc992141", "model": "OpenChat-3.6-8B-20240522", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.371, "raw_score": 37.1, "raw_scale": 100, "source_record": "OpenChat-3.6-8B-20240522", "source_date": "2024-05-22", "report_url": "https://huggingface.co/openchat/openchat-3.6-8b-20240522" }, { "id": "cce44d106c4b3990eee1", "model": "Llama-3-8B-Instruct", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.369, "raw_score": 36.9, "raw_scale": 100, "source_record": "Llama-3-8B-Instruct", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct" }, { "id": "c7c8c0ebacd92289e5dd", "model": "Granite-Code-8B-Base", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.35700000000000004, "raw_score": 35.7, "raw_scale": 100, "source_record": "Granite-Code-8B-Base", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-8b-code-base" }, { "id": "3fbe41841dfcda2d7566", "model": "CodeLlama-34B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.35600000000000004, "raw_score": 35.6, "raw_scale": 100, "source_record": "CodeLlama-34B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf" }, { "id": "ffb1921ef6794e8d0859", "model": "Yi-1.5-9B", "organization": "01.AI", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.355, "raw_score": 35.5, "raw_scale": 100, "source_record": "Yi-1.5-9B", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-9B" }, { "id": "c886958321a7ab6bde93", "model": "Granite-3.0-8B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.354, "raw_score": 35.4, "raw_scale": 100, "source_record": "Granite-3.0-8B-Instruct", "source_date": "2024-10-21", "report_url": "https://huggingface.co/ibm-granite/granite-3.0-8b-instruct" }, { "id": "1cf7cd4c4f5fe21df135", "model": "Yi-1.5-6B-Chat", "organization": "01.AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.33899999999999997, "raw_score": 33.9, "raw_scale": 100, "source_record": "Yi-1.5-6B-Chat", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-6B-Chat" }, { "id": "74c47691997bd779916a", "model": "InternLM2.5-7B-Chat", "organization": "Shanghai AI Laboratory", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.331, "raw_score": 33.1, "raw_scale": 100, "source_record": "InternLM2.5-7B-Chat", "source_date": "2024-07-03", "report_url": "https://huggingface.co/internlm/internlm2_5-7b-chat" }, { "id": "6629c1aa64afd2993f3d", "model": "Qwen2.5-Coder-1.5B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.327, "raw_score": 32.7, "raw_scale": 100, "source_record": "Qwen2.5-Coder-1.5B-Instruct", "source_date": "2024-11-12", "report_url": "https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct" }, { "id": "2a17dc1bb2909ce6fbfb", "model": "CodeLlama-13B-Base", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.32, "raw_score": 32.0, "raw_scale": 100, "source_record": "CodeLlama-13B-Base", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-13b-hf" }, { "id": "52f557045fc8516db57b", "model": "CodeLlama-13B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.317, "raw_score": 31.7, "raw_scale": 100, "source_record": "CodeLlama-13B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf" }, { "id": "b5a25695785d9f388c0e", "model": "Granite-Code-3B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.314, "raw_score": 31.4, "raw_scale": 100, "source_record": "Granite-Code-3B-Instruct", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-3b-code-instruct" }, { "id": "7603f5344a4b9ae2ddec", "model": "DeepSeek-Coder-V2-Lite-Base", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.306, "raw_score": 30.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-V2-Lite-Base", "source_date": "2024-06-17", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Base" }, { "id": "ec7337d7fc279611920e", "model": "OpenCodeInterpreter-DS-1.3B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.298, "raw_score": 29.8, "raw_scale": 100, "source_record": "OpenCodeInterpreter-DS-1.3B", "source_date": "2024-02-22", "report_url": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-1.3B" }, { "id": "921aca304e1a6b4bcc32", "model": "DeepSeek-Coder-1.3B-Instruct", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.29600000000000004, "raw_score": 29.6, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-Instruct", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct" }, { "id": "41a1b9681ffdb82d9c4d", "model": "Llama-3-8B-Base", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.28800000000000003, "raw_score": 28.8, "raw_scale": 100, "source_record": "Llama-3-8B-Base", "source_date": "2024-04-18", "report_url": "https://huggingface.co/meta-llama/Meta-Llama-3-8B" }, { "id": "7a73af90e5907882b901", "model": "CodeLlama-7B-Base", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.287, "raw_score": 28.7, "raw_scale": 100, "source_record": "CodeLlama-7B-Base", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-7b-hf" }, { "id": "cc9e2326b7c1594793c8", "model": "Llama-3.2-3B-Instruct", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.28300000000000003, "raw_score": 28.3, "raw_scale": 100, "source_record": "Llama-3.2-3B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct" }, { "id": "c5b8c76a405f96925998", "model": "StarCoder2-7B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.27699999999999997, "raw_score": 27.7, "raw_scale": 100, "source_record": "StarCoder2-7B", "source_date": "2024-02-29", "report_url": "https://huggingface.co/bigcode/starcoder2-7b" }, { "id": "8a19057d0a8f0bacaa03", "model": "Yi-1.5-6B", "organization": "01.AI", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.275, "raw_score": 27.5, "raw_scale": 100, "source_record": "Yi-1.5-6B", "source_date": "2024-05-20", "report_url": "https://huggingface.co/01-ai/Yi-1.5-6B" }, { "id": "f0039c47cca363813d23", "model": "Mistral-7B-Instruct-v0.3", "organization": "Mistral AI", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.273, "raw_score": 27.3, "raw_scale": 100, "source_record": "Mistral-7B-Instruct-v0.3", "source_date": "2024-05-22", "report_url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3" }, { "id": "54f0332f9b7255ed0edc", "model": "Qwen2.5-1.5B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.271, "raw_score": 27.1, "raw_scale": 100, "source_record": "Qwen2.5-1.5B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct" }, { "id": "27a797d5d20833c33e68", "model": "CodeLlama-7B-Instruct", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.257, "raw_score": 25.7, "raw_scale": 100, "source_record": "CodeLlama-7B-Instruct", "source_date": "2023-08-25", "report_url": "https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf" }, { "id": "825ef89de63469de2a9f", "model": "Granite-Code-20B-Base", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.256, "raw_score": 25.6, "raw_scale": 100, "source_record": "Granite-Code-20B-Base", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-20b-code-base" }, { "id": "8c1dd3ea17320962ada3", "model": "DeepSeek-R1-Distill-Qwen-7B", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.249, "raw_score": 24.9, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-7B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" }, { "id": "c77667958a981959323b", "model": "CodeGemma-2B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.239, "raw_score": 23.9, "raw_scale": 100, "source_record": "CodeGemma-2B", "source_date": "2024-04-10", "report_url": "https://huggingface.co/google/codegemma-2b" }, { "id": "0565dc69d77f2ed642c5", "model": "Mistral-7B-v0.3", "organization": "Mistral AI", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.235, "raw_score": 23.5, "raw_scale": 100, "source_record": "Mistral-7B-v0.3", "source_date": "2024-05-22", "report_url": "https://huggingface.co/mistralai/Mistral-7B-v0.3" }, { "id": "65a1b6072c86b2f1a80a", "model": "DeepSeek-Coder-1.3B-Base", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.222, "raw_score": 22.2, "raw_scale": 100, "source_record": "DeepSeek-Coder-1.3B-Base", "source_date": "2023-10-28", "report_url": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base" }, { "id": "4df7648b2a7ce499a014", "model": "StarCoder2-3B", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.214, "raw_score": 21.4, "raw_scale": 100, "source_record": "StarCoder2-3B", "source_date": "2024-02-29", "report_url": "https://huggingface.co/bigcode/starcoder2-3b" }, { "id": "5e7efb01085ccf1a0ff6", "model": "Granite-Code-3B-Base", "organization": "", "configuration": "Full-Complete; prefill=True; prompted=False", "score": 0.2, "raw_score": 20.0, "raw_scale": 100, "source_record": "Granite-Code-3B-Base", "source_date": "2024-05-06", "report_url": "https://huggingface.co/ibm-granite/granite-3b-code-base" }, { "id": "22fc60302c3cd4a2fd1f", "model": "DeepSeek-R1-Distill-Llama-8B", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.153, "raw_score": 15.3, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Llama-8B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B" }, { "id": "4a3bd480d04ad502597b", "model": "Llama-3.2-1B-Instruct", "organization": "Meta", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.113, "raw_score": 11.3, "raw_scale": 100, "source_record": "Llama-3.2-1B-Instruct", "source_date": "2024-09-25", "report_url": "https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct" }, { "id": "2d399ef44ac721206ab9", "model": "Qwen2.5-0.5B-Instruct", "organization": "Alibaba", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.113, "raw_score": 11.3, "raw_scale": 100, "source_record": "Qwen2.5-0.5B-Instruct", "source_date": "2024-09-19", "report_url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct" }, { "id": "4da4589d256fc2bb8534", "model": "DeepSeek-R1-Distill-Qwen-1.5B", "organization": "DeepSeek", "configuration": "Full-Complete; prefill=True; prompted=True", "score": 0.079, "raw_score": 7.9, "raw_scale": 100, "source_record": "DeepSeek-R1-Distill-Qwen-1.5B", "source_date": "2025-01-20", "report_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" } ] }, { "id": "bird-sql-test-overall", "name": "BIRD-SQL (overall test)", "group": "编程", "unit": "%", "metric": "Execution accuracy", "protocol": { "en": "Full BIRD test execution accuracy on the overall submission board. Each result records the oracle-knowledge setting.", "zh": "BIRD 综合提交榜的全量测试集执行准确率。每项成绩注明是否使用 oracle 知识。" }, "source": { "data_url": "https://bird-bench.github.io/", "sha256": "9701c60f487025c551f391a0e130434702e7bcfe4385a22a4a1bf9b343cdf03c", "retrieved_at": "2026-11-02T05:57:36.721154+00:00", "name": "BIRD", "url": "https://bird-bench.github.io/", "selector": "Overall EX table with Oracle Knowledge column; Test (%)", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "4da37fe62ee0d36db0da", "model": "AskData", "organization": "AT&T CDO - DSAIR", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.8340000000000001, "raw_score": 83.4, "raw_scale": 100, "source_record": "AskData|Overall track; oracle knowledge: ✔️", "source_date": "Oct 22, 2026", "report_url": "https://arxiv.org/abs/2505.19988" }, { "id": "0b2a9cae7fc4421e5a2f", "model": "GrainSQL", "organization": "Sarim Chaudhry, Purdue University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.8295, "raw_score": 82.95, "raw_scale": 100, "source_record": "GrainSQL|Overall track; oracle knowledge: ✔️", "source_date": "Sep 26, 2026" }, { "id": "83ef66d1d886ab0cd5e5", "model": "DataGallery-Text2SQL", "organization": "Huawei 2012 Labs", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.8239, "raw_score": 82.39, "raw_scale": 100, "source_record": "DataGallery-Text2SQL|Overall track; oracle knowledge: ✔️", "source_date": "Sep 07, 2026", "report_url": "https://datagallery.cn/benchmark" }, { "id": "26bf21e420f6495c0cb6", "model": "SiriusAI-SQL", "organization": "Tencent Data Computing Platform Department", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.8228, "raw_score": 82.28, "raw_scale": 100, "source_record": "SiriusAI-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Aug 22, 2026" }, { "id": "ef3356ae8ffa4cc4987b", "model": "Agentar-Scale-SQL", "organization": "Ant Group", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.8167, "raw_score": 81.67, "raw_scale": 100, "source_record": "Agentar-Scale-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Sep 25, 2025", "report_url": "https://arxiv.org/abs/2509.24403" }, { "id": "3f97a00d8666dbfb9c0d", "model": "Sber Text2SQL", "organization": "SberData Research", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.8133, "raw_score": 81.33, "raw_scale": 100, "source_record": "Sber Text2SQL|Overall track; oracle knowledge: ✔️", "source_date": "Jun 19, 2026" }, { "id": "e101e477dc5060336d4e", "model": "Xiaomi Text2SQL", "organization": "Xiaomi ITP & Data", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.8083, "raw_score": 80.83, "raw_scale": 100, "source_record": "Xiaomi Text2SQL|Overall track; oracle knowledge: ✔️", "source_date": "May 27, 2026" }, { "id": "226b9794a2cf71e02a95", "model": "RAS", "organization": "Adya AI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7981999999999999, "raw_score": 79.82, "raw_scale": 100, "source_record": "RAS|Overall track; oracle knowledge: ✔️", "source_date": "Aug 19, 2026" }, { "id": "603187e24e6b38c90f58", "model": "DeepEye", "organization": "HKUST(GZ)", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7909, "raw_score": 79.09, "raw_scale": 100, "source_record": "DeepEye|Overall track; oracle knowledge: ✔️", "source_date": "Jul 14, 2026", "report_url": "https://arxiv.org/pdf/2603.28889" }, { "id": "855cd3b6ae10176f22cc", "model": "MarkovSQL", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.787, "raw_score": 78.7, "raw_scale": 100, "source_record": "MarkovSQL|Overall track; oracle knowledge: ✔️", "source_date": "Jul 04, 2026" }, { "id": "c81c3702a049a48718f6", "model": "DeepEye-SQL", "organization": "HKUST(GZ)", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7842, "raw_score": 78.42, "raw_scale": 100, "source_record": "DeepEye-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Jul 10, 2026", "report_url": "https://arxiv.org/abs/2510.17586" }, { "id": "9e72530376eb261dc3cf", "model": "Spektr-SQL", "organization": "Amazon Ads - SpektrBot", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7831, "raw_score": 78.31, "raw_scale": 100, "source_record": "Spektr-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Jul 11, 2026", "report_url": "https://bird-glass-cc9.notion.site/Spektr-SQL-BIRD-Benchmark-Submission-37da39e6358b806bae30f66ee09fd934" }, { "id": "82a049b4b9d843f24139", "model": "LongData-SQL", "organization": "LongShine AI Research", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7753, "raw_score": 77.53, "raw_scale": 100, "source_record": "LongData-SQL|Overall track; oracle knowledge: ✔️", "source_date": "July 14, 2025" }, { "id": "9b532d4509f22bc6e58a", "model": "AxisSQL", "organization": "UST", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7686, "raw_score": 76.86, "raw_scale": 100, "source_record": "AxisSQL|Overall track; oracle knowledge: ✔️", "source_date": "Aug 03, 2026" }, { "id": "a43747d6de2c04394267", "model": "GT-ChatBI-SQL", "organization": "MR Tech", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.768, "raw_score": 76.8, "raw_scale": 100, "source_record": "GT-ChatBI-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Jul 07, 2026" }, { "id": "a9fcea4128a6b9cdf862", "model": "Zhiwen-Lingsi-Agent", "organization": "China Telecom, TeleAI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7663, "raw_score": 76.63, "raw_scale": 100, "source_record": "Zhiwen-Lingsi-Agent|Overall track; oracle knowledge: ✔️", "source_date": "Jan 02, 2026" }, { "id": "5a91f445f1337811839b", "model": "Q-SQL", "organization": "AWS-Quick Science", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7646999999999999, "raw_score": 76.47, "raw_scale": 100, "source_record": "Q-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Dec 4, 2025", "report_url": "https://www.notion.so/BIRD-v2-Memory-2bfe8bf30514801ba112f92ff20784e3" }, { "id": "97e2fc29f85db843a046", "model": "MIC -SQL", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7641, "raw_score": 76.41, "raw_scale": 100, "source_record": "MIC -SQL|Overall track; oracle knowledge: ✔️", "source_date": "Feb 6, 2026" }, { "id": "26a2a4b4fc784630f425", "model": "DataSpace-Text2SQL", "organization": "Institute of Dataspace, Hefei \nComprehensive National Science Center-IDATA", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7636, "raw_score": 76.36, "raw_scale": 100, "source_record": "DataSpace-Text2SQL|Overall track; oracle knowledge: ✔️", "source_date": "Jul 14, 2026" }, { "id": "e02d1dd3ddf2f77292db", "model": "JoyDataAgent-SQL", "organization": "JD JoyWork & JD-CHO & JDT-RMC", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7619, "raw_score": 76.19, "raw_scale": 100, "source_record": "JoyDataAgent-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Sep 19, 2026" }, { "id": "f755e11dffd2b592758e", "model": "RS", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7608, "raw_score": 76.08, "raw_scale": 100, "source_record": "RS|Overall track; oracle knowledge: ✔️", "source_date": "Sep 07, 2026" }, { "id": "38c656c3efdeb3625372", "model": "CHASE-SQL + Gemini", "organization": "Google Cloud", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7602, "raw_score": 76.02, "raw_scale": 100, "source_record": "CHASE-SQL + Gemini|Overall track; oracle knowledge: ✔️", "source_date": "Apr 16, 2025", "report_url": "https://arxiv.org/abs/2410.01943" }, { "id": "8072dcfca96ca8de22e8", "model": "xiaoyi-text-to-sql", "organization": "wenyuai", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7595999999999999, "raw_score": 75.96, "raw_scale": 100, "source_record": "xiaoyi-text-to-sql|Overall track; oracle knowledge: ✔️", "source_date": "Apr 3, 2026" }, { "id": "005b9072f3d4af9ecb61", "model": "RED-SQL", "organization": "South China Normal University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7591, "raw_score": 75.91, "raw_scale": 100, "source_record": "RED-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Feb 21, 2026" }, { "id": "93bd61b01082f14f5dcb", "model": "Sinovatio-SQL", "organization": "Sinovatio AI Lab", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.758, "raw_score": 75.8, "raw_scale": 100, "source_record": "Sinovatio-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Oct 23, 2025" }, { "id": "55aca0fd0794d84409d7", "model": "TCDataAgent-SQL", "organization": "Tencent Cloud", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7574, "raw_score": 75.74, "raw_scale": 100, "source_record": "TCDataAgent-SQL|Overall track; oracle knowledge: ✔️", "source_date": "May 30, 2025" }, { "id": "d009f4b15e1d1acf03a1", "model": "Contextual-SQL", "organization": "Contextual AI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7563, "raw_score": 75.63, "raw_scale": 100, "source_record": "Contextual-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Feb 27, 2025" }, { "id": "5ca2470175d98bf5635a", "model": "XiYan-SQL", "organization": "Alibaba Cloud", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7563, "raw_score": 75.63, "raw_scale": 100, "source_record": "XiYan-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Dec 17, 2024", "report_url": "https://arxiv.org/abs/2507.04701" }, { "id": "ac25735e423061a71678", "model": "CYAN-SQL", "organization": "Tencent Cloud / Fudan University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7535, "raw_score": 75.35, "raw_scale": 100, "source_record": "CYAN-SQL|Overall track; oracle knowledge: ✔️", "source_date": "May 30, 2025" }, { "id": "2d92ec28bc774b417c69", "model": "DB-SQL", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7535, "raw_score": 75.35, "raw_scale": 100, "source_record": "DB-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Sep 1, 2025" }, { "id": "509606f4c17f8edcf39d", "model": "MCR-SQL + Qwen2.5-Coder-32B-Instruct", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7529, "raw_score": 75.29, "raw_scale": 100, "source_record": "MCR-SQL + Qwen2.5-Coder-32B-Instruct|Overall track; oracle knowledge: ✔️", "source_date": "Jan 8, 2026" }, { "id": "1871f3f2aaee5a6cffc8", "model": "MRS-Agent", "organization": "AIC-Lab, Chungbuk National University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7513, "raw_score": 75.13, "raw_scale": 100, "source_record": "MRS-Agent|Overall track; oracle knowledge: ✔️", "source_date": "May 17, 2026" }, { "id": "1d2160bd84c5e7fec490", "model": "Blender-SQL+Qwen3.8-27B", "organization": "DAMENG", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7451000000000001, "raw_score": 74.51, "raw_scale": 100, "source_record": "Blender-SQL+Qwen3.8-27B|Overall track; oracle knowledge: ✔️", "source_date": "Oct 14, 2026" }, { "id": "2aa525afab1be994758e", "model": "ELG-SQL", "organization": "AMS CIR", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7451000000000001, "raw_score": 74.51, "raw_scale": 100, "source_record": "ELG-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Sep 02, 2026" }, { "id": "2897177d3b112309b8b7", "model": "JT-SQLAgent", "organization": "CMCC, JIUTIAN Research", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7406, "raw_score": 74.06, "raw_scale": 100, "source_record": "JT-SQLAgent|Overall track; oracle knowledge: ✔️", "source_date": "Sep 10, 2025" }, { "id": "91f33918384e37855163", "model": "flash-sqlite: Qwen3.5-2B Generator + Self-Judge (SC@32)", "organization": "Alio Leuchtmann", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7373000000000001, "raw_score": 73.73, "raw_scale": 100, "source_record": "flash-sqlite: Qwen3.5-2B Generator + Self-Judge (SC@32)|Overall track; oracle knowledge: ✔️", "source_date": "Sep 29, 2026" }, { "id": "8ad18081e412e349c5d3", "model": "CSC-SQL + XiYanSQL-QwenCoder-32B-2412", "organization": "Wuhan University of Technology + University of Science and Technology of China", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7367, "raw_score": 73.67, "raw_scale": 100, "source_record": "CSC-SQL + XiYanSQL-QwenCoder-32B-2412|Overall track; oracle knowledge: ✔️", "source_date": "May 27, 2025", "report_url": "https://arxiv.org/abs/2505.13271" }, { "id": "cb2c5c90e8dd37187558", "model": "Kwai-AutoSQL", "organization": "Kuaishou Business Team", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7334, "raw_score": 73.34, "raw_scale": 100, "source_record": "Kwai-AutoSQL|Overall track; oracle knowledge: ✔️", "source_date": "May 18, 2026" }, { "id": "bc202afe0ce7bff718ad", "model": "ExSL + granite-34b-code", "organization": "IBM Research AI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7317, "raw_score": 73.17, "raw_scale": 100, "source_record": "ExSL + granite-34b-code|Overall track; oracle knowledge: ✔️", "source_date": "Oct 27, 2024" }, { "id": "0d4a91470fada8146df6", "model": "Reasoning-SQL 14B", "organization": "Google Cloud / Stanford", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7278, "raw_score": 72.78, "raw_scale": 100, "source_record": "Reasoning-SQL 14B|Overall track; oracle knowledge: ✔️", "source_date": "Mar 31, 2025", "report_url": "https://arxiv.org/pdf/2503.23157" }, { "id": "891943d84eb68747cfc5", "model": "GPT-5.5-xhigh", "organization": "OpenAI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7254999999999999, "raw_score": 72.55, "raw_scale": 100, "source_record": "GPT-5.5-xhigh|Overall track; oracle knowledge: ✔️", "source_date": "Apr 28, 2026" }, { "id": "1f9f068627a0d193c7c6", "model": "GenaSQL", "organization": "Gena Co.", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7228, "raw_score": 72.28, "raw_scale": 100, "source_record": "GenaSQL|Overall track; oracle knowledge: ✔️", "source_date": "Aug 06, 2025", "report_url": "https://arxiv.org/abs/2505.14174" }, { "id": "5468e310a3ef287db29f", "model": "OpenSearch-SQL, v2 + GPT-4o", "organization": "Alibaba Cloud", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7228, "raw_score": 72.28, "raw_scale": 100, "source_record": "OpenSearch-SQL, v2 + GPT-4o|Overall track; oracle knowledge: ✔️", "source_date": "Aug 21, 2024", "report_url": "https://arxiv.org/abs/2502.14913" }, { "id": "0d974c74d87b741256ca", "model": "EGV-SQL + Qwen3.6-27B", "organization": "Tampere University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7216, "raw_score": 72.16, "raw_scale": 100, "source_record": "EGV-SQL + Qwen3.6-27B|Overall track; oracle knowledge: ✔️", "source_date": "Aug 17, 2026" }, { "id": "5e373263732782be6171", "model": "OmniSQL-32B", "organization": "Renmin University of China + ByteDance Infra Lab", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7204999999999999, "raw_score": 72.05, "raw_scale": 100, "source_record": "OmniSQL-32B|Overall track; oracle knowledge: ✔️", "source_date": "Mar 4, 2025", "report_url": "https://arxiv.org/abs/2503.02240" }, { "id": "c0177ad39657e44ec457", "model": "Distillery + GPT-4o", "organization": "Distyl AI Research", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7182999999999999, "raw_score": 71.83, "raw_scale": 100, "source_record": "Distillery + GPT-4o|Overall track; oracle knowledge: ✔️", "source_date": "Jul 22, 2024", "report_url": "https://arxiv.org/abs/2408.07702" }, { "id": "403fe7737f678c25eed8", "model": "Share + GPT-5", "organization": "HKU", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7182999999999999, "raw_score": 71.83, "raw_scale": 100, "source_record": "Share + GPT-5|Overall track; oracle knowledge: ✔️", "source_date": "Sep 22, 2025", "report_url": "https://arxiv.org/abs/2506.00391" }, { "id": "cf757f85ba9f4b37d2e1", "model": "CSC-SQL + Qwen2.5-Coder-7B-Instruct", "organization": "Wuhan University of Technology + University of Science and Technology of China", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7172, "raw_score": 71.72, "raw_scale": 100, "source_record": "CSC-SQL + Qwen2.5-Coder-7B-Instruct|Overall track; oracle knowledge: ✔️", "source_date": "May 27, 2025", "report_url": "https://arxiv.org/abs/2505.13271" }, { "id": "5aefffc454f02ff099d2", "model": "LEAF-SQL", "organization": "JXUFE", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.716, "raw_score": 71.6, "raw_scale": 100, "source_record": "LEAF-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Jun 30, 2025" }, { "id": "be965a72c3efda05dbd4", "model": "Queryosity", "organization": "Conrad Labs", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7116, "raw_score": 71.16, "raw_scale": 100, "source_record": "Queryosity|Overall track; oracle knowledge: ✔️", "source_date": "Apr 10, 2025" }, { "id": "793b4053bfdfd5acf2d7", "model": "CHESS IR + CG + UT", "organization": "Stanford", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.711, "raw_score": 71.1, "raw_scale": 100, "source_record": "CHESS IR + CG + UT|Overall track; oracle knowledge: ✔️", "source_date": "May 21, 2024", "report_url": "https://arxiv.org/abs/2405.16755" }, { "id": "da5a6cb795ef06ccb281", "model": "Infly-RL-SQL-32B", "organization": "inftech.ai", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.706, "raw_score": 70.6, "raw_scale": 100, "source_record": "Infly-RL-SQL-32B|Overall track; oracle knowledge: ✔️", "source_date": "May 08, 2025" }, { "id": "1320a061fb8e44633968", "model": "SLM-SQL + Qwen2.5-Coder-1.5B-Instruct", "organization": "Wuhan University of Technology + University of Science and Technology of China", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7049, "raw_score": 70.49, "raw_scale": 100, "source_record": "SLM-SQL + Qwen2.5-Coder-1.5B-Instruct|Overall track; oracle knowledge: ✔️", "source_date": "Aug 1, 2025", "report_url": "https://arxiv.org/abs/2507.22478" }, { "id": "f9c813282c6e00de97c3", "model": "Alpha-SQL + Qwen2.5-Coder-32B-Instruct", "organization": "HKUST(GZ)", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7026, "raw_score": 70.26, "raw_scale": 100, "source_record": "Alpha-SQL + Qwen2.5-Coder-32B-Instruct|Overall track; oracle knowledge: ✔️", "source_date": "Feb 24, 2025", "report_url": "https://arxiv.org/abs/2502.17248" }, { "id": "ccacafe13722da17c417", "model": "Insights AI", "organization": "Uber Freight", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7026, "raw_score": 70.26, "raw_scale": 100, "source_record": "Insights AI|Overall track; oracle knowledge: ✔️", "source_date": "Aug 28, 2024" }, { "id": "25f5c1350dd49a1e9641", "model": "PURPLE + RED + GPT-4o", "organization": "Fudan University + Transwarp Technology", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7021, "raw_score": 70.21, "raw_scale": 100, "source_record": "PURPLE + RED + GPT-4o|Overall track; oracle knowledge: ✔️", "source_date": "Aug 30, 2024" }, { "id": "f3fd862fe554348dd347", "model": "Claude Opus 4.6", "organization": "Anthropic", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.7015, "raw_score": 70.15, "raw_scale": 100, "source_record": "Claude Opus 4.6|Overall track; oracle knowledge: ✔️", "source_date": "Feb 21, 2026" }, { "id": "344efbcfa7041881ec09", "model": "wmyaccept-SQL", "organization": "Independent Researcher", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6959000000000001, "raw_score": 69.59, "raw_scale": 100, "source_record": "wmyaccept-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Oct 2, 2026" }, { "id": "f13c5ac83e8736b8be5f", "model": "GSR", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6926000000000001, "raw_score": 69.26, "raw_scale": 100, "source_record": "GSR|Overall track; oracle knowledge: ✔️", "source_date": "Jan 9, 2025" }, { "id": "01f8335e7c99be4b5c4f", "model": "PB-SQL, GPT-4o", "organization": "Seoul National University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6926000000000001, "raw_score": 69.26, "raw_scale": 100, "source_record": "PB-SQL, GPT-4o|Overall track; oracle knowledge: ✔️", "source_date": "Nov 10, 2024" }, { "id": "2e536a26d65c49cadcf4", "model": "TC-SQL", "organization": "mindflow.ai", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6920000000000001, "raw_score": 69.2, "raw_scale": 100, "source_record": "TC-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Dec 18, 2024" }, { "id": "379eca335284c95b8dab", "model": "RECAP + Gemini", "organization": "Google Cloud", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6903, "raw_score": 69.03, "raw_scale": 100, "source_record": "RECAP + Gemini|Overall track; oracle knowledge: ✔️", "source_date": "Jul 14, 2024" }, { "id": "4480f38a8ab73153fedc", "model": "XiYanSQL-QwenCoder-32B", "organization": "Alibaba Cloud", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6903, "raw_score": 69.03, "raw_scale": 100, "source_record": "XiYanSQL-QwenCoder-32B|Overall track; oracle knowledge: ✔️", "source_date": "Jan 09, 2025", "report_url": "https://arxiv.org/abs/2507.04701" }, { "id": "723c717b146b419ed7ac", "model": "ByteBrain", "organization": "ByteDance Infra Lab", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6887000000000001, "raw_score": 68.87, "raw_scale": 100, "source_record": "ByteBrain|Overall track; oracle knowledge: ✔️", "source_date": "Jul 2, 2024" }, { "id": "1b9152ce44866cbaf1d1", "model": "RSL-SQL + GPT-4o", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.687, "raw_score": 68.7, "raw_scale": 100, "source_record": "RSL-SQL + GPT-4o|Overall track; oracle knowledge: ✔️", "source_date": "Nov 18, 2024", "report_url": "https://arxiv.org/abs/2411.00073" }, { "id": "214e40e35c27d641be96", "model": "Qwen3.5-2B + StructCoT Distillation (SC@7)", "organization": "Alio Leuchtmann", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6847, "raw_score": 68.47, "raw_scale": 100, "source_record": "Qwen3.5-2B + StructCoT Distillation (SC@7)|Overall track; oracle knowledge: ✔️", "source_date": "Aug 24, 2026" }, { "id": "6e7c86636f05bc6a6329", "model": "Qwen3-Coder-480B-A35B", "organization": "Alibaba", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6814, "raw_score": 68.14, "raw_scale": 100, "source_record": "Qwen3-Coder-480B-A35B|Overall track; oracle knowledge: ✔️", "source_date": "Feb 21, 2026" }, { "id": "65ce7acd76f227bb2d99", "model": "PRIME-SQL", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6803, "raw_score": 68.03, "raw_scale": 100, "source_record": "PRIME-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Aug 06, 2026" }, { "id": "fd08d8bac8abe613448c", "model": "OmniSQL-7B", "organization": "Renmin University of China + ByteDance Infra Lab", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6797, "raw_score": 67.97, "raw_scale": 100, "source_record": "OmniSQL-7B|Overall track; oracle knowledge: ✔️", "source_date": "Mar 4, 2025", "report_url": "https://arxiv.org/abs/2503.02240" }, { "id": "05fbfab228b820a54ea8", "model": "ExSL + granite-20b-code", "organization": "IBM Research AI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6786, "raw_score": 67.86, "raw_scale": 100, "source_record": "ExSL + granite-20b-code|Overall track; oracle knowledge: ✔️", "source_date": "May 14, 2024" }, { "id": "011027f5b4070e5ee5fc", "model": "AskData + GPT-4o", "organization": "AT&T CDO - DSAIR", "configuration": "Overall track; oracle knowledge: ", "score": 0.6740999999999999, "raw_score": 67.41, "raw_scale": 100, "source_record": "AskData + GPT-4o|Overall track; oracle knowledge: ", "source_date": "Nov 11, 2024", "report_url": "https://arxiv.org/abs/2505.19988" }, { "id": "cf11f93c058d19b90e7a", "model": "Claude 4.5 Sonnet", "organization": "Anthropic", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6685, "raw_score": 66.85, "raw_scale": 100, "source_record": "Claude 4.5 Sonnet|Overall track; oracle knowledge: ✔️", "source_date": "Feb 21, 2026" }, { "id": "d92b92d63322330feb1b", "model": "CHESS IR + SS + CG", "organization": "Stanford", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6668999999999999, "raw_score": 66.69, "raw_scale": 100, "source_record": "CHESS IR + SS + CG|Overall track; oracle knowledge: ✔️", "source_date": "May 21, 2024", "report_url": "https://arxiv.org/abs/2405.16755" }, { "id": "895c232c035ee445ec1c", "model": "Elysian-SQL + gpt-oss-120b", "organization": "Independent Researcher", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6646, "raw_score": 66.46, "raw_scale": 100, "source_record": "Elysian-SQL + gpt-oss-120b|Overall track; oracle knowledge: ✔️", "source_date": "Jun 1, 2026" }, { "id": "28a2895ee07cbad8cfd9", "model": "E-SQL + GPT-4o", "organization": "Bilkent University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6629, "raw_score": 66.29, "raw_scale": 100, "source_record": "E-SQL + GPT-4o|Overall track; oracle knowledge: ✔️", "source_date": "Sep 23, 2024" }, { "id": "88995cf13009edb839cc", "model": "Arcwise + GPT-4o", "organization": "Arcwise", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6620999999999999, "raw_score": 66.21, "raw_scale": 100, "source_record": "Arcwise + GPT-4o|Overall track; oracle knowledge: ✔️", "source_date": "Aug 29, 2024" }, { "id": "2e7efb5b18a5f5509927", "model": "NucliOS", "organization": "MathCo", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.659, "raw_score": 65.9, "raw_scale": 100, "source_record": "NucliOS|Overall track; oracle knowledge: ✔️", "source_date": "Sep 10, 2025" }, { "id": "3e8aa30e763828b809c1", "model": "Command A", "organization": "Cohere AI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6568, "raw_score": 65.68, "raw_scale": 100, "source_record": "Command A|Overall track; oracle knowledge: ✔️", "source_date": "Apr 30, 2025", "report_url": "https://arxiv.org/abs/2504.00698" }, { "id": "723b77a63dd018b522cc", "model": "RSL-SQL + DeepSeek-v2", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6551, "raw_score": 65.51, "raw_scale": 100, "source_record": "RSL-SQL + DeepSeek-v2|Overall track; oracle knowledge: ✔️", "source_date": "Nov 18, 2024", "report_url": "https://arxiv.org/abs/2411.00073" }, { "id": "6e87719206aab303438e", "model": "MCS-SQL + GPT-4", "organization": "Dunamu", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6545000000000001, "raw_score": 65.45, "raw_scale": 100, "source_record": "MCS-SQL + GPT-4|Overall track; oracle knowledge: ✔️", "source_date": "Jan 14, 2024", "report_url": "https://arxiv.org/abs/2405.07467" }, { "id": "a2a48c3dfbbcede14a4c", "model": "SCL-SQL", "organization": "Xffuture", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6523, "raw_score": 65.23, "raw_scale": 100, "source_record": "SCL-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Aug 20, 2024" }, { "id": "abd2d43e87d25ec41b05", "model": "OpenSearch-SQL,v1 + GPT-4", "organization": "Alibaba Cloud", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6495000000000001, "raw_score": 64.95, "raw_scale": 100, "source_record": "OpenSearch-SQL,v1 + GPT-4|Overall track; oracle knowledge: ✔️", "source_date": "Apr 08, 2024" }, { "id": "30f47eef0516f447cdbc", "model": "SFT CodeS-15B + SQLFixAgent", "organization": "Soochow University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6462, "raw_score": 64.62, "raw_scale": 100, "source_record": "SFT CodeS-15B + SQLFixAgent|Overall track; oracle knowledge: ✔️", "source_date": "Jun 7, 2024" }, { "id": "09a61145e2044311417d", "model": "PURPLE + GPT-4o", "organization": "Fudan University + Transwarp Technology", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6451, "raw_score": 64.51, "raw_scale": 100, "source_record": "PURPLE + GPT-4o|Overall track; oracle knowledge: ✔️", "source_date": "Aug 30, 2024" }, { "id": "bcea1439d7c0c3cd7590", "model": "MSL-SQL + DeepSeek-V2.5", "organization": "Wuhan University of Technology", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.64, "raw_score": 64.0, "raw_scale": 100, "source_record": "MSL-SQL + DeepSeek-V2.5|Overall track; oracle knowledge: ✔️", "source_date": "Oct 10, 2024" }, { "id": "9c44ae349e7afb0e7cf0", "model": "EBA-SQL + GPT-4", "organization": "Northeastern University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.635, "raw_score": 63.5, "raw_scale": 100, "source_record": "EBA-SQL + GPT-4|Overall track; oracle knowledge: ✔️", "source_date": "Feb 6, 2025" }, { "id": "229b58c9d56991a2d694", "model": "Sense", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6339, "raw_score": 63.39, "raw_scale": 100, "source_record": "Sense|Overall track; oracle knowledge: ✔️", "source_date": "Feb 21, 2024" }, { "id": "58625a643cb0eb6c00b3", "model": "OneSQL-v0.1-Qwen-32B", "organization": "onekq.ai", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6333, "raw_score": 63.33, "raw_scale": 100, "source_record": "OneSQL-v0.1-Qwen-32B|Overall track; oracle knowledge: ✔️", "source_date": "Mar 11, 2025" }, { "id": "80b0303c3bb202b4d8c6", "model": "GRA-SQL", "organization": "Tencent CDP-youpu", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6322, "raw_score": 63.22, "raw_scale": 100, "source_record": "GRA-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Apr 10, 2024" }, { "id": "94eb1c7bbda14a649873", "model": "GLM-4.7", "organization": "Z AI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6294, "raw_score": 62.94, "raw_scale": 100, "source_record": "GLM-4.7|Overall track; oracle knowledge: ✔️", "source_date": "Feb 21, 2026" }, { "id": "591aea342867bd5023f4", "model": "SuperSQL", "organization": "HKUST(GZ)", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6265999999999999, "raw_score": 62.66, "raw_scale": 100, "source_record": "SuperSQL|Overall track; oracle knowledge: ✔️", "source_date": "Jun 1, 2024", "report_url": "https://arxiv.org/abs/2406.01265" }, { "id": "ee41ef233f7f5b3ee0e0", "model": "SLM-SQL + Qwen2.5-Coder-0.5B-Instruct", "organization": "Wuhan University of Technology + University of Science and Technology of China", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6182, "raw_score": 61.82, "raw_scale": 100, "source_record": "SLM-SQL + Qwen2.5-Coder-0.5B-Instruct|Overall track; oracle knowledge: ✔️", "source_date": "Aug 1, 2025", "report_url": "https://arxiv.org/abs/2507.22478" }, { "id": "f53bc60c04cae99a8966", "model": "{Chat2Query} (GPT-4 + data entity modeling) (PingCAP)", "organization": "PingCAP", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6098, "raw_score": 60.98, "raw_scale": 100, "source_record": "{Chat2Query} (GPT-4 + data entity modeling) (PingCAP)|Overall track; oracle knowledge: ✔️", "source_date": "Mar 27, 2024" }, { "id": "c9fd4f4a0c7e5bdca186", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6093, "raw_score": 60.93, "raw_scale": 100, "source_record": "DeepSeek-R1|Overall track; oracle knowledge: ✔️", "source_date": "Feb 21, 2026" }, { "id": "afa69c65cc4dcf0f297b", "model": "Dubo-SQL, v1", "organization": "Mercator Technologies", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6071, "raw_score": 60.71, "raw_scale": 100, "source_record": "Dubo-SQL, v1|Overall track; oracle knowledge: ✔️", "source_date": "Nov 16, 2023" }, { "id": "cb46101173f846f7d15e", "model": "Struct-SQL", "organization": "Crater Labs", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6042000000000001, "raw_score": 60.42, "raw_scale": 100, "source_record": "Struct-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Jan 30, 2026", "report_url": "https://arxiv.org/abs/2512.17053" }, { "id": "1c025909f890af48b74e", "model": "SFT CodeS-15B", "organization": "Renmin University of China", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6037, "raw_score": 60.37, "raw_scale": 100, "source_record": "SFT CodeS-15B|Overall track; oracle knowledge: ✔️", "source_date": "Oct 12, 2023", "report_url": "https://arxiv.org/abs/2402.16347" }, { "id": "a818524582b3ea349403", "model": "DTS-SQL + DeepSeek 7B", "organization": "University of Alberta", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.6031, "raw_score": 60.31, "raw_scale": 100, "source_record": "DTS-SQL + DeepSeek 7B|Overall track; oracle knowledge: ✔️", "source_date": "Feb 27, 2024", "report_url": "https://arxiv.org/abs/2402.01117" }, { "id": "f7a9a855314e0b332490", "model": "Kimi-K2-Thinking", "organization": "Moonshot AI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5987, "raw_score": 59.87, "raw_scale": 100, "source_record": "Kimi-K2-Thinking|Overall track; oracle knowledge: ✔️", "source_date": "Feb 21, 2026" }, { "id": "ac086dc5d91b1ed54df5", "model": "E-SQL + GPT-4o mini", "organization": "Bilkent University", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5981000000000001, "raw_score": 59.81, "raw_scale": 100, "source_record": "E-SQL + GPT-4o mini|Overall track; oracle knowledge: ✔️", "source_date": "Sep 23, 2024" }, { "id": "522a3694f9fa5d17b787", "model": "MAC-SQL + GPT-4", "organization": "BUAA & Tencent", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5959, "raw_score": 59.59, "raw_scale": 100, "source_record": "MAC-SQL + GPT-4|Overall track; oracle knowledge: ✔️", "source_date": "Nov 21, 2023", "report_url": "https://arxiv.org/abs/2312.11242" }, { "id": "540171a14ee30949027c", "model": "xorazm-text2sql-0.8b", "organization": "Martin Luther University Halle-Wittenberg", "configuration": "Overall track; oracle knowledge: ", "score": 0.5959, "raw_score": 59.59, "raw_scale": 100, "source_record": "xorazm-text2sql-0.8b|Overall track; oracle knowledge: ", "source_date": "Jul 17, 2026" }, { "id": "fed6bbedd70f3e2e9fe4", "model": "SFT CodeS-7B", "organization": "Renmin University of China", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5925, "raw_score": 59.25, "raw_scale": 100, "source_record": "SFT CodeS-7B|Overall track; oracle knowledge: ✔️", "source_date": "Oct 12, 2023", "report_url": "https://arxiv.org/abs/2402.16347" }, { "id": "f524efebd45b943301b0", "model": "TA-SQL + GPT-4", "organization": "HKU", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5914, "raw_score": 59.14, "raw_scale": 100, "source_record": "TA-SQL + GPT-4|Overall track; oracle knowledge: ✔️", "source_date": "May 27, 2024", "report_url": "https://arxiv.org/abs/2405.15307" }, { "id": "9d409d4563f0f839794f", "model": "DAIL-SQL + GPT-4", "organization": "Alibaba Group", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5740999999999999, "raw_score": 57.41, "raw_scale": 100, "source_record": "DAIL-SQL + GPT-4|Overall track; oracle knowledge: ✔️", "source_date": "Nov 09, 2023", "report_url": "https://arxiv.org/abs/2308.15363" }, { "id": "424b07b6c51bd8103bb9", "model": "ExSL + granite-20b-code", "organization": "IBM Research AI", "configuration": "Overall track; oracle knowledge: ", "score": 0.5713, "raw_score": 57.13, "raw_scale": 100, "source_record": "ExSL + granite-20b-code|Overall track; oracle knowledge: ", "source_date": "May 24, 2024" }, { "id": "033db59084ef578a3efd", "model": "DeepSeek", "organization": "DeepSeek", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5668, "raw_score": 56.68, "raw_scale": 100, "source_record": "DeepSeek|Overall track; oracle knowledge: ✔️", "source_date": "Aug 10, 2024" }, { "id": "0b1c21c7a6de74a348ec", "model": "DIN-SQL + GPT-4", "organization": "University of Alberta", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5589999999999999, "raw_score": 55.9, "raw_scale": 100, "source_record": "DIN-SQL + GPT-4|Overall track; oracle knowledge: ✔️", "source_date": "Aug 15, 2023", "report_url": "https://arxiv.org/abs/2304.11015" }, { "id": "283ccf4e1c2fae8130c4", "model": "Mistral", "organization": "Mistral AI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5584, "raw_score": 55.84, "raw_scale": 100, "source_record": "Mistral|Overall track; oracle knowledge: ✔️", "source_date": "Aug 08, 2024" }, { "id": "1603e5f8a8080334ddf7", "model": "GPT-4", "organization": "OpenAI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5489, "raw_score": 54.89, "raw_scale": 100, "source_record": "GPT-4|Overall track; oracle knowledge: ✔️", "source_date": "Jul 01, 2023" }, { "id": "b101cfa0458ccf766c7f", "model": "Interactive-T2S", "organization": "Peking University", "configuration": "Overall track; oracle knowledge: ", "score": 0.5411, "raw_score": 54.11, "raw_scale": 100, "source_record": "Interactive-T2S|Overall track; oracle knowledge: ", "source_date": "Nov 8, 2024", "report_url": "https://dl.acm.org/doi/10.1145/3746252.3761052" }, { "id": "39c916c6cdb82ed976ca", "model": "Prem-1B-SQL", "organization": "Prem AI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.5154, "raw_score": 51.54, "raw_scale": 100, "source_record": "Prem-1B-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Sep 19, 2024" }, { "id": "bf6a010273ea374416f9", "model": "Claude-2", "organization": "Anthropic", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.4902, "raw_score": 49.02, "raw_scale": 100, "source_record": "Claude-2|Overall track; oracle knowledge: ✔️", "source_date": "Jul 16, 2023" }, { "id": "95a50b680dab47e35b90", "model": "Open-SQL", "organization": "Anonymous", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.47740000000000005, "raw_score": 47.74, "raw_scale": 100, "source_record": "Open-SQL|Overall track; oracle knowledge: ✔️", "source_date": "Nov 23, 2023" }, { "id": "22360997030720c18546", "model": "ChatGPT + CoT", "organization": "HKU & DAMO", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.4008, "raw_score": 40.08, "raw_scale": 100, "source_record": "ChatGPT + CoT|Overall track; oracle knowledge: ✔️", "source_date": "Mar 17, 2023", "report_url": "https://arxiv.org/abs/2305.03111" }, { "id": "7131d2023671c11a50b5", "model": "ChatGPT", "organization": "OpenAI", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.39299999999999996, "raw_score": 39.3, "raw_scale": 100, "source_record": "ChatGPT|Overall track; oracle knowledge: ✔️", "source_date": "Mar 17, 2023" }, { "id": "cef689c0e98b2eb69d2a", "model": "Codex", "organization": "", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.36469999999999997, "raw_score": 36.47, "raw_scale": 100, "source_record": "Codex|Overall track; oracle knowledge: ✔️", "source_date": "Feb 17, 2023" }, { "id": "c9cc39a6199b5709d1af", "model": "sqlpup (7-sample vote)", "organization": "", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.33149999999999996, "raw_score": 33.15, "raw_scale": 100, "source_record": "sqlpup (7-sample vote)|Overall track; oracle knowledge: ✔️", "source_date": "Aug 24, 2026", "report_url": "https://github.com/shivenkk/sqlpup" }, { "id": "c7e428b2f34b4a343481", "model": "Palm-2", "organization": "Google", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.33039999999999997, "raw_score": 33.04, "raw_scale": 100, "source_record": "Palm-2|Overall track; oracle knowledge: ✔️", "source_date": "Jul 16, 2023" }, { "id": "52d7d76afb55afde1b9c", "model": "ChatGPT + CoT", "organization": "HKU & DAMO", "configuration": "Overall track; oracle knowledge: ", "score": 0.2895, "raw_score": 28.95, "raw_scale": 100, "source_record": "ChatGPT + CoT|Overall track; oracle knowledge: ", "source_date": "Mar 17, 2023", "report_url": "https://arxiv.org/abs/2305.03111" }, { "id": "8ee2f6691aa9573046eb", "model": "sqlpup (single pass)", "organization": "", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.2733, "raw_score": 27.33, "raw_scale": 100, "source_record": "sqlpup (single pass)|Overall track; oracle knowledge: ✔️", "source_date": "Aug 24, 2026", "report_url": "https://github.com/shivenkk/sqlpup" }, { "id": "ece16adf274b89cd3795", "model": "ChatGPT", "organization": "OpenAI", "configuration": "Overall track; oracle knowledge: ", "score": 0.2677, "raw_score": 26.77, "raw_scale": 100, "source_record": "ChatGPT|Overall track; oracle knowledge: ", "source_date": "Mar 17, 2023" }, { "id": "d7466a76d24a14305ed5", "model": "Codex", "organization": "", "configuration": "Overall track; oracle knowledge: ", "score": 0.2486, "raw_score": 24.86, "raw_scale": 100, "source_record": "Codex|Overall track; oracle knowledge: ", "source_date": "Feb 17, 2023" }, { "id": "3c099bb515773f5da846", "model": "T5-3B", "organization": "", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.24050000000000002, "raw_score": 24.05, "raw_scale": 100, "source_record": "T5-3B|Overall track; oracle knowledge: ✔️", "source_date": "Feb 5, 2023" }, { "id": "f78b275eae68279b820d", "model": "T5-Large", "organization": "", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.2094, "raw_score": 20.94, "raw_scale": 100, "source_record": "T5-Large|Overall track; oracle knowledge: ✔️", "source_date": "Feb 3, 2023" }, { "id": "40fddd85da9fbb9622d4", "model": "T5-Base", "organization": "", "configuration": "Overall track; oracle knowledge: ✔️", "score": 0.12890000000000001, "raw_score": 12.89, "raw_scale": 100, "source_record": "T5-Base|Overall track; oracle knowledge: ✔️", "source_date": "Feb 3, 2023" }, { "id": "9d9ccb84128e59d0d3dc", "model": "T5-3B", "organization": "", "configuration": "Overall track; oracle knowledge: ", "score": 0.1117, "raw_score": 11.17, "raw_scale": 100, "source_record": "T5-3B|Overall track; oracle knowledge: ", "source_date": "Feb 5, 2023" }, { "id": "7978153916b93feec63c", "model": "T5-Large", "organization": "", "configuration": "Overall track; oracle knowledge: ", "score": 0.1038, "raw_score": 10.38, "raw_scale": 100, "source_record": "T5-Large|Overall track; oracle knowledge: ", "source_date": "Feb 3, 2023" }, { "id": "f8cef372702b9145e13f", "model": "T5-Base", "organization": "", "configuration": "Overall track; oracle knowledge: ", "score": 0.0706, "raw_score": 7.06, "raw_scale": 100, "source_record": "T5-Base|Overall track; oracle knowledge: ", "source_date": "Feb 3, 2023" } ] }, { "id": "spider-2-snow", "name": "Spider 2.0-Snow", "group": "编程", "unit": "%", "metric": "Task success", "protocol": { "en": "Official Snow submission track, 547 tasks. Model and agent names follow the authors’ table.", "zh": "官方 Snow 提交赛道，共 547 道任务。模型与 Agent 名称取自作者的原表。" }, "source": { "data_url": "https://spider2-sql.github.io/", "sha256": "3cd4bb95ed0f2d9d5e3aa76cb2461f9f22c8af8ac7b7d6136e3589ce01474e61", "retrieved_at": "2026-11-02T05:57:36.441642+00:00", "name": "Spider 2.0", "url": "https://spider2-sql.github.io/", "selector": "#ex-table-snow Score / 100; latest dated identical submission", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "e74cdb4ef7447b8a1e83", "model": "Genloop's Sentinel Agent v2 Pro", "organization": "Genloop", "configuration": "Spider 2.0-Snow", "score": 0.9670000000000001, "raw_score": 96.7, "raw_scale": 100, "source_record": "Genloop's Sentinel Agent v2 Pro|Spider 2.0-Snow", "source_date": "Mar 1, 2026", "report_url": "https://genloop.ai/" }, { "id": "f5990fb03a8783e76225", "model": "Native mini", "organization": "usenative.ai", "configuration": "Spider 2.0-Snow", "score": 0.9653, "raw_score": 96.53, "raw_scale": 100, "source_record": "Native mini|Spider 2.0-Snow", "source_date": "Feb 18, 2026" }, { "id": "314e72d4c9cd28674278", "model": "QUVI-3 + Gemini-3-pro-preview", "organization": "DAQUV", "configuration": "Spider 2.0-Snow", "score": 0.9415, "raw_score": 94.15, "raw_scale": 100, "source_record": "QUVI-3 + Gemini-3-pro-preview|Spider 2.0-Snow", "source_date": "Feb 9, 2026" }, { "id": "eacfeb8d3d1e7ce801bc", "model": "TCDataAgent-SQL with Contextual Scaling Engine", "organization": "Tencent Cloud Big Data", "configuration": "Spider 2.0-Snow", "score": 0.9397, "raw_score": 93.97, "raw_scale": 100, "source_record": "TCDataAgent-SQL with Contextual Scaling Engine|Spider 2.0-Snow", "source_date": "Feb 3, 2026" }, { "id": "eb5c8ed65e5bcfcb522b", "model": "Prism Swarm with Deepthink + Claude-Sonnet-4.5", "organization": "Paytm", "configuration": "Spider 2.0-Snow", "score": 0.9048999999999999, "raw_score": 90.49, "raw_scale": 100, "source_record": "Prism Swarm with Deepthink + Claude-Sonnet-4.5|Spider 2.0-Snow", "source_date": "Jan 27, 2026", "report_url": "https://github.com/paytm/prism" }, { "id": "a494c48170a55f694e75", "model": "Genloop's Sentinel Agent v2", "organization": "Genloop", "configuration": "Spider 2.0-Snow", "score": 0.8848, "raw_score": 88.48, "raw_scale": 100, "source_record": "Genloop's Sentinel Agent v2|Spider 2.0-Snow", "source_date": "Feb 17, 2026", "report_url": "https://genloop.ai/" }, { "id": "7787b02322aaba4b1d8d", "model": "Ask Data with Relational Knowledge Graph", "organization": "AT&T CDO & RelationalAI", "configuration": "Spider 2.0-Snow", "score": 0.8628, "raw_score": 86.28, "raw_scale": 100, "source_record": "Ask Data with Relational Knowledge Graph|Spider 2.0-Snow", "source_date": "Jan 7, 2026" }, { "id": "8eb9677d7f6b3b68ace3", "model": "QUVI-3 + Claude-Opus-4.6", "organization": "DAQUV", "configuration": "Spider 2.0-Snow", "score": 0.8628, "raw_score": 86.28, "raw_scale": 100, "source_record": "QUVI-3 + Claude-Opus-4.6|Spider 2.0-Snow", "source_date": "Feb 6, 2026" }, { "id": "344d9ede76b697237c9c", "model": "ByteBrain-Agent", "organization": "ByteDance Infra System Lab", "configuration": "Spider 2.0-Snow", "score": 0.841, "raw_score": 84.1, "raw_scale": 100, "source_record": "ByteBrain-Agent|Spider 2.0-Snow", "source_date": "Dec 16, 2025" }, { "id": "446e9cacf062e7bec388", "model": "MAESTRO — Multi-Agent Engine for Semantic T2S Reasoning & Orchestration", "organization": "IntelligentOps Tigers", "configuration": "Spider 2.0-Snow", "score": 0.8373, "raw_score": 83.73, "raw_scale": 100, "source_record": "MAESTRO — Multi-Agent Engine for Semantic T2S Reasoning & Orchestration|Spider 2.0-Snow", "source_date": "May 17, 2026" }, { "id": "da2aa04f23d6cba5ccc9", "model": "Genloop's Sentinel Agent v1.5", "organization": "Genloop", "configuration": "Spider 2.0-Snow", "score": 0.8336, "raw_score": 83.36, "raw_scale": 100, "source_record": "Genloop's Sentinel Agent v1.5|Spider 2.0-Snow", "source_date": "Feb 13, 2026", "report_url": "https://genloop.ai/" }, { "id": "ed28da9b47363cb04542", "model": "Deepinsight Agent", "organization": "Ant Group", "configuration": "Spider 2.0-Snow", "score": 0.83, "raw_score": 83.0, "raw_scale": 100, "source_record": "Deepinsight Agent|Spider 2.0-Snow", "source_date": "May 21, 2026" }, { "id": "2ef6505e3748efbc2c85", "model": "AiCheng Agent", "organization": "alibaba_cfo_tech", "configuration": "Spider 2.0-Snow", "score": 0.8281000000000001, "raw_score": 82.81, "raw_scale": 100, "source_record": "AiCheng Agent|Spider 2.0-Snow", "source_date": "Jan 9, 2026" }, { "id": "d3865163609dc9566527", "model": "Omni 2.0", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.8281000000000001, "raw_score": 82.81, "raw_scale": 100, "source_record": "Omni 2.0|Spider 2.0-Snow", "source_date": "Jul 8, 2026" }, { "id": "9ef8c0690eb7296da649", "model": "Prism Swarm + Claude-Sonnet-4.5", "organization": "Paytm", "configuration": "Spider 2.0-Snow", "score": 0.8262999999999999, "raw_score": 82.63, "raw_scale": 100, "source_record": "Prism Swarm + Claude-Sonnet-4.5|Spider 2.0-Snow", "source_date": "Jan 2, 2026", "report_url": "https://github.com/paytm/prism" }, { "id": "386ef0f3424c2f41bc0f", "model": "LingXi Agent + Claude-Sonnet-4.5", "organization": "Ant Group", "configuration": "Spider 2.0-Snow", "score": 0.7989, "raw_score": 79.89, "raw_scale": 100, "source_record": "LingXi Agent + Claude-Sonnet-4.5|Spider 2.0-Snow", "source_date": "Dec 5, 2025" }, { "id": "b11c2937c26aa6f79dd0", "model": "Arctic-FLEX", "organization": "Snowflake AI Research", "configuration": "Spider 2.0-Snow", "score": 0.7514, "raw_score": 75.14, "raw_scale": 100, "source_record": "Arctic-FLEX|Spider 2.0-Snow", "source_date": "Dec 9, 2025" }, { "id": "de9011fac000150df09e", "model": "Sophon-Agent", "organization": "ByteDance DataPlatform LLM", "configuration": "Spider 2.0-Snow", "score": 0.7404000000000001, "raw_score": 74.04, "raw_scale": 100, "source_record": "Sophon-Agent|Spider 2.0-Snow", "source_date": "Nov 24, 2025" }, { "id": "34c1c7bf59fc9e2cd716", "model": "APEX-SQL", "organization": "LIGHTSPEED", "configuration": "Spider 2.0-Snow", "score": 0.7313, "raw_score": 73.13, "raw_scale": 100, "source_record": "APEX-SQL|Spider 2.0-Snow", "source_date": "Feb 28, 2026", "report_url": "https://arxiv.org/pdf/2602.16720" }, { "id": "ce05ee7620c222e38ed6", "model": "QiSi-SQL + Deepseek3.2", "organization": "Ant Group Tech Risk", "configuration": "Spider 2.0-Snow", "score": 0.7038, "raw_score": 70.38, "raw_scale": 100, "source_record": "QiSi-SQL + Deepseek3.2|Spider 2.0-Snow", "source_date": "Dec 26, 2025" }, { "id": "ee9b44fc0ee2cbfe0545", "model": "PExA", "organization": "Bloomberg - AI Engineering group", "configuration": "Spider 2.0-Snow", "score": 0.7020000000000001, "raw_score": 70.2, "raw_scale": 100, "source_record": "PExA|Spider 2.0-Snow", "source_date": "Sep 26, 2025" }, { "id": "00da71cd0037b69bb3bf", "model": "DIA (Data Intelligence Agents)", "organization": "C3 AI", "configuration": "Spider 2.0-Snow", "score": 0.6947, "raw_score": 69.47, "raw_scale": 100, "source_record": "DIA (Data Intelligence Agents)|Spider 2.0-Snow", "source_date": "May 27, 2026", "report_url": "https://c3.ai/" }, { "id": "ec7fc7a662314039b86c", "model": "Chicory AI Agent + Claude Sonnet 4.5 + Opus 4.5 Judge", "organization": "Chicory AI", "configuration": "Spider 2.0-Snow", "score": 0.6728000000000001, "raw_score": 67.28, "raw_scale": 100, "source_record": "Chicory AI Agent + Claude Sonnet 4.5 + Opus 4.5 Judge|Spider 2.0-Snow", "source_date": "Dec 4, 2025", "report_url": "https://app.chicory.ai" }, { "id": "036e7fcb83b074dfe88e", "model": "Omni 1.0", "organization": "360 Labs", "configuration": "Spider 2.0-Snow", "score": 0.6636, "raw_score": 66.36, "raw_scale": 100, "source_record": "Omni 1.0|Spider 2.0-Snow", "source_date": "Jul 2, 2026" }, { "id": "f700766937d5ef3505f6", "model": "SSDAT + GPT-5", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.6563, "raw_score": 65.63, "raw_scale": 100, "source_record": "SSDAT + GPT-5|Spider 2.0-Snow", "source_date": "Oct 2, 2025" }, { "id": "d3347223e93733b9a6eb", "model": "FlexSQL + gpt-oss-120b", "organization": "University of Alberta", "configuration": "Spider 2.0-Snow", "score": 0.6545000000000001, "raw_score": 65.45, "raw_scale": 100, "source_record": "FlexSQL + gpt-oss-120b|Spider 2.0-Snow", "source_date": "May 6, 2025", "report_url": "https://arxiv.org/abs/2605.02815" }, { "id": "2cd5c8ab228c7c29e3e3", "model": "ReFoRCE + o3", "organization": "Hao AI Lab x Snowflake", "configuration": "Spider 2.0-Snow", "score": 0.6289, "raw_score": 62.89, "raw_scale": 100, "source_record": "ReFoRCE + o3|Spider 2.0-Snow", "source_date": "Nov 3, 2025", "report_url": "https://arxiv.org/abs/2502.00675" }, { "id": "89df45442af043589569", "model": "WindAgent + Claude-4-Sonnet", "organization": "MeiTuan AI For FinData", "configuration": "Spider 2.0-Snow", "score": 0.6143, "raw_score": 61.43, "raw_scale": 100, "source_record": "WindAgent + Claude-4-Sonnet|Spider 2.0-Snow", "source_date": "Aug 8, 2025" }, { "id": "07853461f68a31219f09", "model": "PAI-DataSurfer Agent", "organization": "Alibaba Cloud Computing Platform", "configuration": "Spider 2.0-Snow", "score": 0.6033, "raw_score": 60.33, "raw_scale": 100, "source_record": "PAI-DataSurfer Agent|Spider 2.0-Snow", "source_date": "Sep 16, 2025" }, { "id": "7b0dc5ce3e604e774d18", "model": "AutoLink + DeepSeek-R1", "organization": "HUST VLR Lab", "configuration": "Spider 2.0-Snow", "score": 0.5484, "raw_score": 54.84, "raw_scale": 100, "source_record": "AutoLink + DeepSeek-R1|Spider 2.0-Snow", "source_date": "Jul 19, 2025", "report_url": "https://arxiv.org/abs/2511.17190" }, { "id": "6bef91a037196e7983fa", "model": "GD-TTS Agent + GPT-5.4-mini", "organization": "Gagandeep Singh", "configuration": "Spider 2.0-Snow", "score": 0.5046, "raw_score": 50.46, "raw_scale": 100, "source_record": "GD-TTS Agent + GPT-5.4-mini|Spider 2.0-Snow", "source_date": "May 29, 2026" }, { "id": "5f3a65adf87040f67d67", "model": "PGV-Agent + GLM-5", "organization": "Yiwei, Bangguo", "configuration": "Spider 2.0-Snow", "score": 0.5027, "raw_score": 50.27, "raw_scale": 100, "source_record": "PGV-Agent + GLM-5|Spider 2.0-Snow", "source_date": "Mar 4, 2026" }, { "id": "b86e8b4784de7ebb3721", "model": "Meituan-agent", "organization": "Meituan FinData Intelligence", "configuration": "Spider 2.0-Snow", "score": 0.4534, "raw_score": 45.34, "raw_scale": 100, "source_record": "Meituan-agent|Spider 2.0-Snow", "source_date": "Jul 7, 2025" }, { "id": "4fb483f4cb7c8a1f0b30", "model": "KDGCCloud-KCILab + Qwen3-Max", "organization": "KDGCCloud-KCILab Team", "configuration": "Spider 2.0-Snow", "score": 0.4515, "raw_score": 45.15, "raw_scale": 100, "source_record": "KDGCCloud-KCILab + Qwen3-Max|Spider 2.0-Snow", "source_date": "Feb 26, 2026" }, { "id": "33276c872b96d50378d1", "model": "AgenticView + GPT-5-mini", "organization": "Griffith BigData Lab, Griffith University", "configuration": "Spider 2.0-Snow", "score": 0.40950000000000003, "raw_score": 40.95, "raw_scale": 100, "source_record": "AgenticView + GPT-5-mini|Spider 2.0-Snow", "source_date": "Dec 8, 2025" }, { "id": "47588a534533e3fc7c0b", "model": "CT-ChatBI + Qwen3-Max", "organization": "CT-ChatBI Team", "configuration": "Spider 2.0-Snow", "score": 0.40399999999999997, "raw_score": 40.4, "raw_scale": 100, "source_record": "CT-ChatBI + Qwen3-Max|Spider 2.0-Snow", "source_date": "Mar 30, 2026" }, { "id": "41339d12f4edb25374b5", "model": "Chat2DB-Agent + Claude-4-Sonnet", "organization": "Chat2DB", "configuration": "Spider 2.0-Snow", "score": 0.3839, "raw_score": 38.39, "raw_scale": 100, "source_record": "Chat2DB-Agent + Claude-4-Sonnet|Spider 2.0-Snow", "source_date": "Jul 2, 2025" }, { "id": "f4bc048ab331f51ce335", "model": "ReFoRCE + DeepSeek-V3", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.3803, "raw_score": 38.03, "raw_scale": 100, "source_record": "ReFoRCE + DeepSeek-V3|Spider 2.0-Snow", "source_date": "Dev 2, 2025" }, { "id": "68d55c3d16172bb03fa7", "model": "Spider-Agent + Qwen3-Coder-Plus", "organization": "Bot-UTAI", "configuration": "Spider 2.0-Snow", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "Spider-Agent + Qwen3-Coder-Plus|Spider 2.0-Snow", "source_date": "Feb 9, 2026" }, { "id": "087c8b644a58d005f3a2", "model": "ReFoRCE + o1-preview", "organization": "Hao AI Lab x Snowflake", "configuration": "Spider 2.0-Snow", "score": 0.3126, "raw_score": 31.26, "raw_scale": 100, "source_record": "ReFoRCE + o1-preview|Spider 2.0-Snow", "source_date": "Jan 28, 2025", "report_url": "https://arxiv.org/abs/2502.00675" }, { "id": "0ec028c23edbcf03face", "model": "Spider-Agent + Qwen3-Coder", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.31079999999999997, "raw_score": 31.08, "raw_scale": 100, "source_record": "Spider-Agent + Qwen3-Coder|Spider 2.0-Snow", "source_date": "Jul 28, 2025" }, { "id": "d4c25550bda83771cdb0", "model": "Spider-Agent + Claude-4-Sonnet-20250514", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.25780000000000003, "raw_score": 25.78, "raw_scale": 100, "source_record": "Spider-Agent + Claude-4-Sonnet-20250514|Spider 2.0-Snow", "source_date": "Mar 26, 2025" }, { "id": "7c087eda97b4c4fe8f23", "model": "Spider-Agent + Claude-3.7-Sonnet-20250219", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.245, "raw_score": 24.5, "raw_scale": 100, "source_record": "Spider-Agent + Claude-3.7-Sonnet-20250219|Spider 2.0-Snow", "source_date": "Mar 8, 2025" }, { "id": "e67ed396afd237bba873", "model": "Spider-Agent + Claude-3.7-Sonnet-20250219-Thinking", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.24309999999999998, "raw_score": 24.31, "raw_scale": 100, "source_record": "Spider-Agent + Claude-3.7-Sonnet-20250219-Thinking|Spider 2.0-Snow", "source_date": "Mar 16, 2025" }, { "id": "085896052ea707903b79", "model": "Spider-Agent + o1-preview", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.23579999999999998, "raw_score": 23.58, "raw_scale": 100, "source_record": "Spider-Agent + o1-preview|Spider 2.0-Snow", "source_date": "Nov 30, 2024" }, { "id": "d7726604d6c809927fc1", "model": "Spider-Agent + o1-2024-12-17", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.2321, "raw_score": 23.21, "raw_scale": 100, "source_record": "Spider-Agent + o1-2024-12-17|Spider 2.0-Snow", "source_date": "Feb 11, 2025" }, { "id": "fb5c078c7ddbcffaef28", "model": "Spider-Agent + o3-mini-2025-01-31", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.192, "raw_score": 19.2, "raw_scale": 100, "source_record": "Spider-Agent + o3-mini-2025-01-31|Spider 2.0-Snow", "source_date": "Feb 1, 2025" }, { "id": "6705807af24b9c204a4f", "model": "Spider-Agent + Claude-3.5-Sonnet-20241022", "organization": "AWS ProServe", "configuration": "Spider 2.0-Snow", "score": 0.19010000000000002, "raw_score": 19.01, "raw_scale": 100, "source_record": "Spider-Agent + Claude-3.5-Sonnet-20241022|Spider 2.0-Snow", "source_date": "Mar 7, 2025" }, { "id": "df94360feb06b0298d7d", "model": "Spider-Agent + Gemini-2.0-Pro", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.1389, "raw_score": 13.89, "raw_scale": 100, "source_record": "Spider-Agent + Gemini-2.0-Pro|Spider 2.0-Snow", "source_date": "Mar 13, 2025" }, { "id": "ceca87da3d14257ef1bc", "model": "AI-DIVE + gpt-oss-120B", "organization": "S-Core", "configuration": "Spider 2.0-Snow", "score": 0.131, "raw_score": 13.1, "raw_scale": 100, "source_record": "AI-DIVE + gpt-oss-120B|Spider 2.0-Snow", "source_date": "Apr 20, 2026", "report_url": "https://s-core.co.kr/en/" }, { "id": "0731aaf3fc9207949ad7", "model": "Spider-Agent + GPT-4o-2024-11-20", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.1298, "raw_score": 12.98, "raw_scale": 100, "source_record": "Spider-Agent + GPT-4o-2024-11-20|Spider 2.0-Snow", "source_date": "Nov 30, 2024" }, { "id": "d93689ba6fc00344fdc9", "model": "Spider-Agent + DeepSeek-R1", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.10550000000000001, "raw_score": 10.55, "raw_scale": 100, "source_record": "Spider-Agent + DeepSeek-R1|Spider 2.0-Snow", "source_date": "Mar 8, 2025" }, { "id": "abf2266b0241dfa806ba", "model": "CollideNL2SQL + GPT-4o", "organization": "Collide Tech", "configuration": "Spider 2.0-Snow", "score": 0.0968, "raw_score": 9.68, "raw_scale": 100, "source_record": "CollideNL2SQL + GPT-4o|Spider 2.0-Snow", "source_date": "Jan 3, 2025" }, { "id": "8f24025c432055762919", "model": "Spider-Agent + QwQ-32B", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.08960000000000001, "raw_score": 8.96, "raw_scale": 100, "source_record": "Spider-Agent + QwQ-32B|Spider 2.0-Snow", "source_date": "Mar 10, 2025" }, { "id": "59fc09d22d0e37dd275b", "model": "Spider-Agent + DeepSeek-V3", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.08779999999999999, "raw_score": 8.78, "raw_scale": 100, "source_record": "Spider-Agent + DeepSeek-V3|Spider 2.0-Snow", "source_date": "Jan 20, 2025" }, { "id": "8f8a5c1da8a5e347a7e4", "model": "Spider-Agent + Qwen2.5-Coder-32B-Instruct", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.0548, "raw_score": 5.48, "raw_scale": 100, "source_record": "Spider-Agent + Qwen2.5-Coder-32B-Instruct|Spider 2.0-Snow", "source_date": "Nov 30, 2024" }, { "id": "c17da8668618f9b682c1", "model": "Dail-SQL + GPT-4o", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.022000000000000002, "raw_score": 2.2, "raw_scale": 100, "source_record": "Dail-SQL + GPT-4o|Spider 2.0-Snow", "source_date": "Nov 2, 2024" }, { "id": "2a84ad617be4c7f20354", "model": "CHESS + GPT-4o", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.0128, "raw_score": 1.28, "raw_scale": 100, "source_record": "CHESS + GPT-4o|Spider 2.0-Snow", "source_date": "Nov 2, 2024" }, { "id": "43babfc157e77f9f3a8c", "model": "DIN-SQL + GPT-4o", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "DIN-SQL + GPT-4o|Spider 2.0-Snow", "source_date": "Nov 2, 2024" }, { "id": "2cef87d9433452652ca2", "model": "SFT CodeS-15B", "organization": "", "configuration": "Spider 2.0-Snow", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "SFT CodeS-15B|Spider 2.0-Snow", "source_date": "Nov 2, 2024" } ] }, { "id": "spider-2-dbt", "name": "Spider 2.0-DBT", "group": "编程", "unit": "%", "metric": "Task success", "protocol": { "en": "Official DBT submission track, 68 tasks. Model and agent names follow the authors’ table.", "zh": "官方 DBT 提交赛道，共 68 道任务。模型与 Agent 名称取自作者的原表。" }, "source": { "data_url": "https://spider2-sql.github.io/", "sha256": "3cd4bb95ed0f2d9d5e3aa76cb2461f9f22c8af8ac7b7d6136e3589ce01474e61", "retrieved_at": "2026-11-02T05:57:36.441642+00:00", "name": "Spider 2.0", "url": "https://spider2-sql.github.io/", "selector": "#ex-table-main Score / 100; latest dated identical submission", "status": "current_snapshot", "kind": "official_submissions" }, "results": [ { "id": "c11bcd9c30c0443bb1e5", "model": "SignalPilot Agent", "organization": "signalpilot.ai", "configuration": "Spider 2.0-DBT", "score": 0.6559999999999999, "raw_score": 65.6, "raw_scale": 100, "source_record": "SignalPilot Agent|Spider 2.0-DBT", "source_date": "May 29, 2026", "report_url": "https://github.com/SignalPilot-Labs/SignalPilot/" }, { "id": "041c89ddff7d7436f189", "model": "Databao Agent", "organization": "Databao by JetBrains", "configuration": "Spider 2.0-DBT", "score": 0.6029, "raw_score": 60.29, "raw_scale": 100, "source_record": "Databao Agent|Spider 2.0-DBT", "source_date": "May 11, 2026", "report_url": "https://databao.app/" }, { "id": "e3c871ec063323967338", "model": "Shadowfax-DBT-Agent + GPT-5", "organization": "Shadowfax Data", "configuration": "Spider 2.0-DBT", "score": 0.4118, "raw_score": 41.18, "raw_scale": 100, "source_record": "Shadowfax-DBT-Agent + GPT-5|Spider 2.0-DBT", "source_date": "Jul 26, 2025" }, { "id": "31016b14d42eab965fef", "model": "Spider-Agent-Extended + GPT-5", "organization": "Woods, A. '25.", "configuration": "Spider 2.0-DBT", "score": 0.3971, "raw_score": 39.71, "raw_scale": 100, "source_record": "Spider-Agent-Extended + GPT-5|Spider 2.0-DBT", "source_date": "Aug 9, 2025" }, { "id": "2b991b793b8e09a7d2c1", "model": "USTC-KCIL DBT Agent + GPT-5.4", "organization": "USTC-KCIL", "configuration": "Spider 2.0-DBT", "score": 0.3971, "raw_score": 39.71, "raw_scale": 100, "source_record": "USTC-KCIL DBT Agent + GPT-5.4|Spider 2.0-DBT", "source_date": "Jun 17, 2026" }, { "id": "1e4077667a8b34b709ef", "model": "DIA (Data Intelligence Agents)", "organization": "C3 AI", "configuration": "Spider 2.0-DBT", "score": 0.375, "raw_score": 37.5, "raw_scale": 100, "source_record": "DIA (Data Intelligence Agents)|Spider 2.0-DBT", "source_date": "May 27, 2026", "report_url": "https://c3.ai/" }, { "id": "0a83e9e5d67984bf252d", "model": "Chicory AI Agent + Claude Sonnet 4.5", "organization": "Chicory AI", "configuration": "Spider 2.0-DBT", "score": 0.3529, "raw_score": 35.29, "raw_scale": 100, "source_record": "Chicory AI Agent + Claude Sonnet 4.5|Spider 2.0-DBT", "source_date": "Dec 4, 2025", "report_url": "https://app.chicory.ai" }, { "id": "344909d73d316f5abd73", "model": "Spider-Agent-DBT + GPT-5.4", "organization": "", "configuration": "Spider 2.0-DBT", "score": 0.3529, "raw_score": 35.29, "raw_scale": 100, "source_record": "Spider-Agent-DBT + GPT-5.4|Spider 2.0-DBT", "source_date": "May 20, 2026" }, { "id": "a6315e0b26b34c69aa15", "model": "Symbiote Agent + GPT-5", "organization": "Databao by JetBrains", "configuration": "Spider 2.0-DBT", "score": 0.3529, "raw_score": 35.29, "raw_scale": 100, "source_record": "Symbiote Agent + GPT-5|Spider 2.0-DBT", "source_date": "Oct 29, 2025" }, { "id": "33f06ae30a7417d1103e", "model": "DAQUV_QUVI_Agent", "organization": "DAQUV", "configuration": "Spider 2.0-DBT", "score": 0.1765, "raw_score": 17.65, "raw_scale": 100, "source_record": "DAQUV_QUVI_Agent|Spider 2.0-DBT", "source_date": "Jun 25, 2025", "report_url": "https://daquv.com/en" }, { "id": "0d9ce9c2fae32c7d4f0a", "model": "Spider-Agent + Claude-3.7-Sonnet-20250219", "organization": "", "configuration": "Spider 2.0-DBT", "score": 0.147, "raw_score": 14.7, "raw_scale": 100, "source_record": "Spider-Agent + Claude-3.7-Sonnet-20250219|Spider 2.0-DBT", "source_date": "May 22, 2025" }, { "id": "9ba73d0488a6dd45dadc", "model": "Spider-Agent + o1-preview", "organization": "", "configuration": "Spider 2.0-DBT", "score": 0.1324, "raw_score": 13.24, "raw_scale": 100, "source_record": "Spider-Agent + o1-preview|Spider 2.0-DBT", "source_date": "Nov 2, 2024" }, { "id": "ba0c871c2d554cb63e5b", "model": "Spider-Agent + GPT-4.1", "organization": "", "configuration": "Spider 2.0-DBT", "score": 0.10289999999999999, "raw_score": 10.29, "raw_scale": 100, "source_record": "Spider-Agent + GPT-4.1|Spider 2.0-DBT", "source_date": "Jun 26, 2025" }, { "id": "423ef7e06eb20a90465b", "model": "Spider-Agent + GPT-4o", "organization": "", "configuration": "Spider 2.0-DBT", "score": 0.0735, "raw_score": 7.35, "raw_scale": 100, "source_record": "Spider-Agent + GPT-4o|Spider 2.0-DBT", "source_date": "Nov 2, 2024" }, { "id": "08a566b0ceda69c71c23", "model": "Spider-Agent + o3-mini", "organization": "", "configuration": "Spider 2.0-DBT", "score": 0.0441, "raw_score": 4.41, "raw_scale": 100, "source_record": "Spider-Agent + o3-mini|Spider 2.0-DBT", "source_date": "Jun 26, 2025" }, { "id": "c0bc67a473e57c6b9ebb", "model": "Spider-Agent + o3", "organization": "", "configuration": "Spider 2.0-DBT", "score": 0.0294, "raw_score": 2.94, "raw_scale": 100, "source_record": "Spider-Agent + o3|Spider 2.0-DBT", "source_date": "Jun 26, 2025" } ] }, { "id": "cmmlu-0shot", "name": "CMMLU (0-shot)", "group": "知识与推理", "unit": "%", "metric": "Average accuracy", "protocol": { "en": "Author-maintained zero-shot table. Each score is the published average across subjects.", "zh": "作者零样本榜单，各项成绩为原表中的跨学科平均准确率。" }, "source": { "data_url": "https://raw.githubusercontent.com/haonan-li/CMMLU/master/README.md", "sha256": "958e0b90d4258bb477b6f432d8eeb919148339efa8869f83e71a60d36ef3e24c", "retrieved_at": "2026-11-02T05:57:45.363667+00:00", "name": "CMMLU authors", "url": "https://github.com/haonan-li/CMMLU", "selector": "README Zero-shot section: average column; exclude Random", "status": "current_snapshot", "kind": "official_leaderboard" }, "results": [ { "id": "436292affe45caaf8cde", "model": "Spark 4.0-2024-10-14", "organization": "", "configuration": "0-shot; publisher README", "score": 0.9097, "raw_score": 90.97, "raw_scale": 100, "source_record": "Spark 4.0-2024-10-14|0-shot; publisher README" }, { "id": "8be7adcdf70f18ff70fc", "model": "Telechat2-35B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.9048999999999999, "raw_score": 90.49, "raw_scale": 100, "source_record": "Telechat2-35B|0-shot; publisher README" }, { "id": "4dd3eef389d85ba7d49c", "model": "Lingzhi-72B-chat", "organization": "", "configuration": "0-shot; publisher README", "score": 0.9007, "raw_score": 90.07, "raw_scale": 100, "source_record": "Lingzhi-72B-chat|0-shot; publisher README" }, { "id": "ad93fc49e5df54738f1e", "model": "Qwen1.5-110B", "organization": "Alibaba", "configuration": "0-shot; publisher README", "score": 0.8764, "raw_score": 87.64, "raw_scale": 100, "source_record": "Qwen1.5-110B|0-shot; publisher README" }, { "id": "a69bb701e14c9166f1e9", "model": "Qwen2-72B", "organization": "Alibaba", "configuration": "0-shot; publisher README", "score": 0.8747, "raw_score": 87.47, "raw_scale": 100, "source_record": "Qwen2-72B|0-shot; publisher README" }, { "id": "b91b9ba94225501c74b5", "model": "Qwen2.5-72B", "organization": "Alibaba", "configuration": "0-shot; publisher README", "score": 0.847, "raw_score": 84.7, "raw_scale": 100, "source_record": "Qwen2.5-72B|0-shot; publisher README" }, { "id": "6b0d80b2d653a71aa96c", "model": "BlueLM", "organization": "", "configuration": "0-shot; publisher README", "score": 0.8468000000000001, "raw_score": 84.68, "raw_scale": 100, "source_record": "BlueLM|0-shot; publisher README" }, { "id": "3098791500cd1abacc45", "model": "DiMind", "organization": "", "configuration": "0-shot; publisher README", "score": 0.8273, "raw_score": 82.73, "raw_scale": 100, "source_record": "DiMind|0-shot; publisher README" }, { "id": "f9c7fa699827479e53c3", "model": "PCI-TransGPT", "organization": "", "configuration": "0-shot; publisher README", "score": 0.8244, "raw_score": 82.44, "raw_scale": 100, "source_record": "PCI-TransGPT|0-shot; publisher README" }, { "id": "8d7d544f02e54142cb0e", "model": "Qwen1.5-72B", "organization": "Alibaba", "configuration": "0-shot; publisher README", "score": 0.8181, "raw_score": 81.81, "raw_scale": 100, "source_record": "Qwen1.5-72B|0-shot; publisher README" }, { "id": "98a54da3ae5829523201", "model": "Qwen1.5-32B", "organization": "Alibaba", "configuration": "0-shot; publisher README", "score": 0.8147, "raw_score": 81.47, "raw_scale": 100, "source_record": "Qwen1.5-32B|0-shot; publisher README" }, { "id": "f2dbd8e227dc1c94502a", "model": "Youyuanjian", "organization": "", "configuration": "0-shot; publisher README", "score": 0.8119, "raw_score": 81.19, "raw_scale": 100, "source_record": "Youyuanjian|0-shot; publisher README" }, { "id": "0384a9dbd6d71dd78180", "model": "云天天书", "organization": "", "configuration": "0-shot; publisher README", "score": 0.8062, "raw_score": 80.62, "raw_scale": 100, "source_record": "云天天书|0-shot; publisher README" }, { "id": "8a94a62771ff2e6e6e3f", "model": "ZhiLu-2-8B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.7995, "raw_score": 79.95, "raw_scale": 100, "source_record": "ZhiLu-2-8B|0-shot; publisher README" }, { "id": "42bf582e3cc731cb5781", "model": "Mind GPT", "organization": "", "configuration": "0-shot; publisher README", "score": 0.792, "raw_score": 79.2, "raw_scale": 100, "source_record": "Mind GPT|0-shot; publisher README" }, { "id": "e5a972279005b5ba4822", "model": "QuarkLLM", "organization": "", "configuration": "0-shot; publisher README", "score": 0.7707999999999999, "raw_score": 77.08, "raw_scale": 100, "source_record": "QuarkLLM|0-shot; publisher README" }, { "id": "a481241878f7eee18d8a", "model": "BlueLM-7B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.754, "raw_score": 75.4, "raw_scale": 100, "source_record": "BlueLM-7B|0-shot; publisher README" }, { "id": "d6a28401c97b826f5648", "model": "Galaxy", "organization": "", "configuration": "0-shot; publisher README", "score": 0.7384999999999999, "raw_score": 73.85, "raw_scale": 100, "source_record": "Galaxy|0-shot; publisher README" }, { "id": "cb1d7c6411f1b05401a1", "model": "ZW-LM", "organization": "", "configuration": "0-shot; publisher README", "score": 0.7273999999999999, "raw_score": 72.74, "raw_scale": 100, "source_record": "ZW-LM|0-shot; publisher README" }, { "id": "bab36557c2352f5150db", "model": "Qwen1.5-7B", "organization": "Alibaba", "configuration": "0-shot; publisher README", "score": 0.7105, "raw_score": 71.05, "raw_scale": 100, "source_record": "Qwen1.5-7B|0-shot; publisher README" }, { "id": "b5b88528607f1a2ab303", "model": "XuanYuan-70B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.7059000000000001, "raw_score": 70.59, "raw_scale": 100, "source_record": "XuanYuan-70B|0-shot; publisher README" }, { "id": "eed9ea49306ef516f39f", "model": "KwaiYii-66B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.6995999999999999, "raw_score": 69.96, "raw_scale": 100, "source_record": "KwaiYii-66B|0-shot; publisher README" }, { "id": "e177f704ced97e4a706e", "model": "Llama-3.1-70B-Instruct", "organization": "Meta", "configuration": "0-shot; publisher README", "score": 0.6901, "raw_score": 69.01, "raw_scale": 100, "source_record": "Llama-3.1-70B-Instruct|0-shot; publisher README" }, { "id": "2f78c4b7fd6a6d384ecf", "model": "GPT4", "organization": "OpenAI", "configuration": "0-shot; publisher README", "score": 0.6890000000000001, "raw_score": 68.9, "raw_scale": 100, "source_record": "GPT4|0-shot; publisher README" }, { "id": "5b64b9832fa9a0d49400", "model": "Mengzi-7B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.6688, "raw_score": 66.88, "raw_scale": 100, "source_record": "Mengzi-7B|0-shot; publisher README" }, { "id": "ef1b0e7dffb5d86d8d2a", "model": "Llama-3-70B-Instruct", "organization": "Meta", "configuration": "0-shot; publisher README", "score": 0.6674, "raw_score": 66.74, "raw_scale": 100, "source_record": "Llama-3-70B-Instruct|0-shot; publisher README" }, { "id": "bd7e4b84ea4631377b19", "model": "KwaiYii-13B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.6122, "raw_score": 61.22, "raw_scale": 100, "source_record": "KwaiYii-13B|0-shot; publisher README" }, { "id": "c891dc93a66a87eb2a2d", "model": "FanFan-1.5B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.6103000000000001, "raw_score": 61.03, "raw_scale": 100, "source_record": "FanFan-1.5B|0-shot; publisher README" }, { "id": "3a1e39b2b6c06277b40f", "model": "XuanYuan-13B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.6051, "raw_score": 60.51, "raw_scale": 100, "source_record": "XuanYuan-13B|0-shot; publisher README" }, { "id": "78d908a425c8ae280a65", "model": "MiLM-6B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.6037, "raw_score": 60.37, "raw_scale": 100, "source_record": "MiLM-6B|0-shot; publisher README" }, { "id": "4150ae0ca7eb8baa1a2f", "model": "Qwen-7B", "organization": "Alibaba", "configuration": "0-shot; publisher README", "score": 0.5757, "raw_score": 57.57, "raw_scale": 100, "source_record": "Qwen-7B|0-shot; publisher README" }, { "id": "d16d3aad16ec054cbaa1", "model": "ZhiLu-13B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.5714, "raw_score": 57.14, "raw_scale": 100, "source_record": "ZhiLu-13B|0-shot; publisher README" }, { "id": "c1610042e8fda2e1f8e9", "model": "Baichuan-13B", "organization": "Baichuan", "configuration": "0-shot; publisher README", "score": 0.5463, "raw_score": 54.63, "raw_scale": 100, "source_record": "Baichuan-13B|0-shot; publisher README" }, { "id": "a55293cf909936d25bba", "model": "ChatGPT", "organization": "OpenAI", "configuration": "0-shot; publisher README", "score": 0.5322, "raw_score": 53.22, "raw_scale": 100, "source_record": "ChatGPT|0-shot; publisher README" }, { "id": "18949056d3f6c9a38293", "model": "MiLM-1.3B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.5079, "raw_score": 50.79, "raw_scale": 100, "source_record": "MiLM-1.3B|0-shot; publisher README" }, { "id": "a6f8026755946e32b093", "model": "ChatGLM2-6B", "organization": "Z AI", "configuration": "0-shot; publisher README", "score": 0.49950000000000006, "raw_score": 49.95, "raw_scale": 100, "source_record": "ChatGLM2-6B|0-shot; publisher README" }, { "id": "55aef04d94d563f6669d", "model": "BLOOMZ-7B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.428, "raw_score": 42.8, "raw_scale": 100, "source_record": "BLOOMZ-7B|0-shot; publisher README" }, { "id": "4a3c6d42c4fc41b1ce67", "model": "Baichuan-7B", "organization": "Baichuan", "configuration": "0-shot; publisher README", "score": 0.4233, "raw_score": 42.33, "raw_scale": 100, "source_record": "Baichuan-7B|0-shot; publisher README" }, { "id": "d4067267ff1bce38218b", "model": "ChatGLM-6B", "organization": "Z AI", "configuration": "0-shot; publisher README", "score": 0.4079, "raw_score": 40.79, "raw_scale": 100, "source_record": "ChatGLM-6B|0-shot; publisher README" }, { "id": "7038922d8c0e483deac5", "model": "BatGPT-15B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.3851, "raw_score": 38.51, "raw_scale": 100, "source_record": "BatGPT-15B|0-shot; publisher README" }, { "id": "ab7176edfec279203fee", "model": "Falcon-40B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.385, "raw_score": 38.5, "raw_scale": 100, "source_record": "Falcon-40B|0-shot; publisher README" }, { "id": "09ee6c8468abf9b1fcef", "model": "LLaMA-65B", "organization": "Meta", "configuration": "0-shot; publisher README", "score": 0.3488, "raw_score": 34.88, "raw_scale": 100, "source_record": "LLaMA-65B|0-shot; publisher README" }, { "id": "215f9e163e196ef005ba", "model": "Bactrian-LLaMA-13B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.3006, "raw_score": 30.06, "raw_scale": 100, "source_record": "Bactrian-LLaMA-13B|0-shot; publisher README" }, { "id": "3fba079b3cccccdbba0c", "model": "Chinese-LLaMA-13B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.2734, "raw_score": 27.34, "raw_scale": 100, "source_record": "Chinese-LLaMA-13B|0-shot; publisher README" }, { "id": "32f46bb7e8e756901ff2", "model": "MOSS-SFT-16B", "organization": "", "configuration": "0-shot; publisher README", "score": 0.2688, "raw_score": 26.88, "raw_scale": 100, "source_record": "MOSS-SFT-16B|0-shot; publisher README" } ] }, { "id": "gaia2-vanilla-2025", "name": "Gaia2 (Vanilla, 2025)", "group": "Agent 与工具", "unit": "%", "metric": "pass@1", "protocol": { "en": "Meta’s September 2025 reference runs on original Gaia2, including noise and agent-to-agent scenarios. Each score is the published macro success rate.", "zh": "Meta 于 2025 年 9 月公布的原始 Gaia2 基线，含噪声与 Agent 间交互场景。成绩取各场景的宏平均成功率。" }, "source": { "data_url": "https://meta-agents-research-environments-leaderboard.hf.space/config", "sha256": "40439a261f217030b71b0735c5cc1776ca4b63802a36c9beb81373cd9b20b1e0", "retrieved_at": "2026-11-02T05:57:45.657175+00:00", "name": "Meta / Gaia2", "url": "https://huggingface.co/spaces/meta-agents-research-environments/leaderboard", "selector": "Vanilla Dataframe: pass@1 = macro_success_rate; Submitter=Meta", "status": "dated_release", "kind": "official_leaderboard" }, "results": [ { "id": "88b8bbff5647c929d43d", "model": "GPT-5 (high)", "organization": "OpenAI", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.42100000000000004, "raw_score": 42.1, "raw_scale": 100, "source_record": "GPT-5 (high)|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "592414fc809d517b020a", "model": "Claude-4-Sonnet Thinking", "organization": "Anthropic", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "Claude-4-Sonnet Thinking|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-29", "reported_by": "Meta" }, { "id": "f4c7f6f08385c9953539", "model": "Claude-4-Sonnet", "organization": "Anthropic", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.348, "raw_score": 34.8, "raw_scale": 100, "source_record": "Claude-4-Sonnet|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "f23ac6a665fd5bc561b8", "model": "GPT-5 (low)", "organization": "OpenAI", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.34600000000000003, "raw_score": 34.6, "raw_scale": 100, "source_record": "GPT-5 (low)|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "1a67ac124fd9d965f7c4", "model": "Gemini-2.5-Pro", "organization": "Google", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.258, "raw_score": 25.8, "raw_scale": 100, "source_record": "Gemini-2.5-Pro|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "6f720d5a6d078e5800ab", "model": "DeepSeek-v3.1 Terminus", "organization": "DeepSeek", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.231, "raw_score": 23.1, "raw_scale": 100, "source_record": "DeepSeek-v3.1 Terminus|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-30", "reported_by": "Meta" }, { "id": "0bb2ec78409153a51d7c", "model": "DeepSeek-v3.1", "organization": "DeepSeek", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.21899999999999997, "raw_score": 21.9, "raw_scale": 100, "source_record": "DeepSeek-v3.1|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-29", "reported_by": "Meta" }, { "id": "1156432891dc53fa77f8", "model": "Kimi-K2", "organization": "Moonshot", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.201, "raw_score": 20.1, "raw_scale": 100, "source_record": "Kimi-K2|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "18abb5af79af307ac278", "model": "GPT-5 (minimal)", "organization": "OpenAI", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.182, "raw_score": 18.2, "raw_scale": 100, "source_record": "GPT-5 (minimal)|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "e1c2748789e38645f887", "model": "Grok-4", "organization": "xAI", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.157, "raw_score": 15.7, "raw_scale": 100, "source_record": "Grok-4|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "f4f424c2df28a07f1db4", "model": "Qwen3-235B-thinking", "organization": "Alibaba", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.157, "raw_score": 15.7, "raw_scale": 100, "source_record": "Qwen3-235B-thinking|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-29", "reported_by": "Meta" }, { "id": "b9b82ef313b5e2ae2106", "model": "GPT-OSS 120B (high)", "organization": "OpenAI", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.13699999999999998, "raw_score": 13.7, "raw_scale": 100, "source_record": "GPT-OSS 120B (high)|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-29", "reported_by": "Meta" }, { "id": "a47d3bfb1caae7bca000", "model": "Qwen3-235B", "organization": "Alibaba", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.11599999999999999, "raw_score": 11.6, "raw_scale": 100, "source_record": "Qwen3-235B|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "c00fb5b88b875adb1a51", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.07400000000000001, "raw_score": 7.4, "raw_scale": 100, "source_record": "GPT-4o|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "a2878cfc97af23038167", "model": "Llama 4 Maverick", "organization": "Meta", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.07400000000000001, "raw_score": 7.4, "raw_scale": 100, "source_record": "Llama 4 Maverick|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" }, { "id": "7dec0a6eebf4a4256783", "model": "Llama 3.3 70B Instruct", "organization": "Meta", "configuration": "Vanilla Gaia2; Meta reference agent", "score": 0.044000000000000004, "raw_score": 4.4, "raw_scale": 100, "source_record": "Llama 3.3 70B Instruct|Vanilla Gaia2; Meta reference agent", "source_date": "2025-09-09", "reported_by": "Meta" } ], "public_history": true }, { "id": "mmlu-qwen25-base-2024", "name": "MMLU (Qwen2.5 base comparison 2024, 5-shot)", "group": "知识与推理", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Qwen2.5 base-model evaluation, 5-shot. The four model-size tables use one published protocol.", "zh": "Qwen2.5 发布时的基础模型对比，5 样本。四张按模型规模分组的表沿用同一评测设置。" }, "source": { "data_url": "https://qwenlm.github.io/blog/qwen2.5-llm/", "sha256": "0d5651d2c4e49d3dd58c3988127d04a14e787cda9d7f9e4881e1f6a4d3f44fef", "retrieved_at": "2026-11-02T06:10:16.746553+00:00", "name": "Qwen Team", "url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation", "kind": "model_report", "status": "dated_release", "selector": "Qwen2.5 Base Language Model Evaluation; all four size tables; exact row MMLU (5-shot)" }, "results": [ { "id": "9ff4f076981550119798", "model": "Qwen2.5-72B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.861, "raw_score": 86.1, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2.5-72B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "eb2e9cf1a8e83b73634c", "model": "Llama-3-405B", "organization": "Meta", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.852, "raw_score": 85.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Llama-3-405B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "764589ae6da67b37910b", "model": "Qwen2-72B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.8420000000000001, "raw_score": 84.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2-72B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "54d7743208864fd56f77", "model": "Qwen2.5-32B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.833, "raw_score": 83.3, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2.5-32B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "6433b442444edce5f1c8", "model": "Qwen2.5-14B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.797, "raw_score": 79.7, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2.5-14B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "692707c402238e241a17", "model": "Llama-3-70B", "organization": "Meta", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.795, "raw_score": 79.5, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Llama-3-70B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "f3a10eeda4c91ead69d1", "model": "Mixtral-8x22B", "organization": "Mistral AI", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.778, "raw_score": 77.8, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Mixtral-8x22B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "a7e4f1c84d40064bf818", "model": "Yi-1.5-34B", "organization": "01.AI", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.772, "raw_score": 77.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Yi-1.5-34B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "36a6a9cc39a871e2ddee", "model": "Qwen2-57B-A14B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.765, "raw_score": 76.5, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2-57B-A14B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "4327e5428d15ddc21a92", "model": "Gemma2-27B", "organization": "Google", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.752, "raw_score": 75.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Gemma2-27B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "556b6947d34d16836a59", "model": "Qwen1.5-32B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.743, "raw_score": 74.3, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen1.5-32B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "89d943ac32bfe55fdb20", "model": "Qwen2.5-7B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.742, "raw_score": 74.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2.5-7B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "9e44fda563d86d2632b8", "model": "Gemma2-9B", "organization": "Google", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.713, "raw_score": 71.3, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Gemma2-9B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "453eba9866b4ec7aa1a7", "model": "Qwen2-7B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.703, "raw_score": 70.3, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2-7B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "65cfb79b580717f11e18", "model": "Llama3-8B", "organization": "Meta", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.6659999999999999, "raw_score": 66.6, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Llama3-8B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "89f26b4aa47b819de23c", "model": "Qwen2.5-3B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.6559999999999999, "raw_score": 65.6, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2.5-3B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "842ac9fd7a97ceca5278", "model": "Mistral-7B", "organization": "Mistral AI", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.642, "raw_score": 64.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Mistral-7B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "4d927330ab58748d3f6a", "model": "Qwen2.5-1.5B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.609, "raw_score": 60.9, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2.5-1.5B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "d85df2edbfe02c7b6a6b", "model": "Qwen2-1.5B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.5589999999999999, "raw_score": 55.9, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2-1.5B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "5190b8fc09bdf87bcefb", "model": "Gemma2-2.6B", "organization": "Google", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.522, "raw_score": 52.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Gemma2-2.6B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "12e0ada22896bc4912c2", "model": "Qwen2.5-0.5B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.475, "raw_score": 47.5, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2.5-0.5B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "d7db3aa00c2163b70c2c", "model": "Qwen2-0.5B", "organization": "Alibaba", "configuration": "Base; 5-shot; Qwen2.5 release", "score": 0.44299999999999995, "raw_score": 44.3, "raw_scale": 100, "source_record": "Qwen2.5|Base|MMLU|Qwen2-0.5B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" } ], "public_history": true }, { "id": "gsm8k-qwen25-base-2024", "name": "GSM8K (Qwen2.5 base comparison 2024, 4-shot)", "group": "数学与科学", "unit": "%", "metric": "Accuracy", "protocol": { "en": "Qwen2.5 base-model evaluation, 4-shot. The four model-size tables use one published protocol.", "zh": "Qwen2.5 发布时的基础模型对比，4 样本。四张按模型规模分组的表沿用同一评测设置。" }, "source": { "data_url": "https://qwenlm.github.io/blog/qwen2.5-llm/", "sha256": "0d5651d2c4e49d3dd58c3988127d04a14e787cda9d7f9e4881e1f6a4d3f44fef", "retrieved_at": "2026-11-02T06:10:16.746553+00:00", "name": "Qwen Team", "url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation", "kind": "model_report", "status": "dated_release", "selector": "Qwen2.5 Base Language Model Evaluation; all four size tables; exact row GSM8K (4-shot)" }, "results": [ { "id": "b27df0cdd1efc1fcec9c", "model": "Qwen2.5-32B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.929, "raw_score": 92.9, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2.5-32B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "e14dcc50314ac9737df8", "model": "Qwen2.5-72B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.915, "raw_score": 91.5, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2.5-72B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "6a1b56ec1d6aadbf69e6", "model": "Qwen2.5-14B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.902, "raw_score": 90.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2.5-14B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "07db34f09ce0edee3cf6", "model": "Llama-3-405B", "organization": "Meta", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.89, "raw_score": 89.0, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Llama-3-405B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "07a9173d5a322a7bfb13", "model": "Qwen2-72B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.89, "raw_score": 89.0, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2-72B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "24b6f0c7eb7c8af3f83c", "model": "Qwen2.5-7B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.8540000000000001, "raw_score": 85.4, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2.5-7B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "0101a95ad2a175eb1e3b", "model": "Mixtral-8x22B", "organization": "Mistral AI", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.8370000000000001, "raw_score": 83.7, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Mixtral-8x22B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "e2c74d6e47e003f3aa25", "model": "Yi-1.5-34B", "organization": "01.AI", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.8170000000000001, "raw_score": 81.7, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Yi-1.5-34B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "de9efe591a043d0631e9", "model": "Gemma2-27B", "organization": "Google", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.8109999999999999, "raw_score": 81.1, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Gemma2-27B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "9b3a60cd911f43c1db80", "model": "Qwen2-57B-A14B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.807, "raw_score": 80.7, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2-57B-A14B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "dd75fc6d1531bc97d182", "model": "Qwen2-7B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.802, "raw_score": 80.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2-7B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "6609884ef9098afe8ac5", "model": "Qwen2.5-3B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.7909999999999999, "raw_score": 79.1, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2.5-3B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "aa4f6a74ba44f0d5f2b9", "model": "Qwen1.5-32B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.785, "raw_score": 78.5, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen1.5-32B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "b5ffb442953e3ea114c3", "model": "Llama-3-70B", "organization": "Meta", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.7759999999999999, "raw_score": 77.6, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Llama-3-70B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "e10ab7b48514e254400b", "model": "Gemma2-9B", "organization": "Google", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.7070000000000001, "raw_score": 70.7, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Gemma2-9B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "4d858163d5faa744d557", "model": "Qwen2.5-1.5B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.685, "raw_score": 68.5, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2.5-1.5B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "07909bb8c70234e3201f", "model": "Llama3-8B", "organization": "Meta", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.5529999999999999, "raw_score": 55.3, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Llama3-8B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "51dcbe667e3e4f9260fe", "model": "Qwen2-1.5B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.469, "raw_score": 46.9, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2-1.5B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "c71edfc92aa1bc1d24d4", "model": "Qwen2.5-0.5B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.41600000000000004, "raw_score": 41.6, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2.5-0.5B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "cc29ad61d4604e5fcddf", "model": "Qwen2-0.5B", "organization": "Alibaba", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.364, "raw_score": 36.4, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Qwen2-0.5B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "1954fe2f179f499e76f6", "model": "Mistral-7B", "organization": "Mistral AI", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.36200000000000004, "raw_score": 36.2, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Mistral-7B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" }, { "id": "175d0910990f15349a58", "model": "Gemma2-2.6B", "organization": "Google", "configuration": "Base; 4-shot; Qwen2.5 release", "score": 0.303, "raw_score": 30.3, "raw_scale": 100, "source_record": "Qwen2.5|Base|GSM8K|Gemma2-2.6B", "source_date": "2024-09-19", "report_url": "https://qwenlm.github.io/blog/qwen2.5-llm/#qwen25-base-language-model-evaluation" } ], "public_history": true }, { "id": "apex-accounting-mercor", "name": "APEX-Accounting", "group": "专业工作", "unit": "score", "metric": "mean-score", "protocol": { "en": "Mercor global leaderboard; mean rubric score on 0–100. Each model/reasoning/harness configuration remains separate.", "zh": "Mercor 总榜；0–100 的量规平均分。各模型、推理与 Agent 配置分别记录。" }, "source": { "name": "Mercor", "benchmark_creator": "Mercor / Ramp", "url": "https://www.mercor.com/apex/apex-accounting-leaderboard/", "data_url": "https://www.mercor.com/apex/", "sha256": "b78316b4f963ace8a9365e6551de468728cb6180d19b961e1cdfc43e83f45ad6", "retrieved_at": "2026-11-02T05:58:01.181119+00:00", "selector": "__NEXT_DATA__.props.pageProps.benchmarks[benchmarkId=apex-accounting].globalLeaderboard; mean-score; harnessScores", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://www.mercor.com/apex/methodology/", "sha256": "4efca79d43944b2dc7dae6a4e6a19d282af66380ff11ab267590ee819eb586b4", "retrieved_at": "2026-11-02T06:12:25.248577+00:00" } ] }, "results": [ { "id": "4675e186742ef507b973", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 71.5, "raw_score": 71.5, "raw_scale": 1, "source_record": "gemini-4-argon|loop_agent|mean-score", "source_model_id": "gemini-4-argon", "n_samples": null, "reported_error": 4.8 }, { "id": "a2a2f206d540eafc0fa8", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 70.7, "raw_score": 70.7, "raw_scale": 1, "source_record": "gemini-4-carbon|loop_agent|mean-score", "source_model_id": "gemini-4-carbon", "n_samples": 400, "reported_error": 4.8 }, { "id": "e0885e8a248eb536d180", "model": "Muse Wonder 1.0", "organization": "Meta", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 67.1, "raw_score": 67.1, "raw_scale": 1, "source_record": "muse-wonder-1-0-max|loop_agent|mean-score", "source_model_id": "muse-wonder-1-0-max", "n_samples": null, "reported_error": 4.9 }, { "id": "254453abe331ef330333", "model": "Fable 5.1", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 66.7, "raw_score": 66.7, "raw_scale": 1, "source_record": "claude-fable-5.1|loop_agent|mean-score", "source_model_id": "claude-fable-5.1", "n_samples": null, "reported_error": 4.6 }, { "id": "f1221ce532db22a5874c", "model": "Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 65.5, "raw_score": 65.5, "raw_scale": 1, "source_record": "claude-opus-5-5|loop_agent|mean-score", "source_model_id": "claude-opus-5-5", "n_samples": null, "reported_error": 4.8 }, { "id": "d9d2f11f0841a41bdb13", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 64.9, "raw_score": 64.9, "raw_scale": 1, "source_record": "muse-spark-1-3-max|loop_agent|mean-score", "source_model_id": "muse-spark-1-3-max", "n_samples": null, "reported_error": 4.8 }, { "id": "d20e5b170e7923d46e0f", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 64.7, "raw_score": 64.7, "raw_scale": 1, "source_record": "gpt-6-astra-max|loop_agent|mean-score", "source_model_id": "gpt-6-astra-max", "n_samples": null, "reported_error": 4.8 }, { "id": "e75fd9bbbecc5fda48bc", "model": "Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 64.5, "raw_score": 64.5, "raw_scale": 1, "source_record": "claude-sonnet-5-5|loop_agent|mean-score", "source_model_id": "claude-sonnet-5-5", "n_samples": null, "reported_error": 4.6 }, { "id": "54f491056c318aadfc8c", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 64.3, "raw_score": 64.3, "raw_scale": 1, "source_record": "gpt-6-1-sol|loop_agent|mean-score", "source_model_id": "gpt-6-1-sol", "n_samples": null, "reported_error": 4.1 }, { "id": "110218b0c963528b31d8", "model": "Opus 5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 64.3, "raw_score": 64.3, "raw_scale": 1, "source_record": "claude-opus-5|loop_agent|mean-score", "source_model_id": "claude-opus-5", "n_samples": null, "reported_error": 5 }, { "id": "84f6d1ffc52814ea9cec", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 63.7, "raw_score": 63.7, "raw_scale": 1, "source_record": "gpt-6-1-terra|loop_agent|mean-score", "source_model_id": "gpt-6-1-terra", "n_samples": null, "reported_error": 5 }, { "id": "82f9258043650c622484", "model": "Kimi K3.1 Pro", "organization": "Kimi", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 62.4, "raw_score": 62.4, "raw_scale": 1, "source_record": "kimi-k3.1-pro|loop_agent|mean-score", "source_model_id": "kimi-k3.1-pro", "n_samples": 400, "reported_error": 4.8 }, { "id": "fa390135f030272788c5", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 62.2, "raw_score": 62.2, "raw_scale": 1, "source_record": "gpt-5-6-sol-max-pro|loop_agent|mean-score", "source_model_id": "gpt-5-6-sol-max-pro", "n_samples": null, "reported_error": 4.9 }, { "id": "94206b4d6d7f4f160996", "model": "Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 62.2, "raw_score": 62.2, "raw_scale": 1, "source_record": "claude-haiku-5-5|loop_agent|mean-score", "source_model_id": "claude-haiku-5-5", "n_samples": null, "reported_error": 4.9 }, { "id": "9e973a7b2e3da8f57f18", "model": "Fable 5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 61.0, "raw_score": 61.0, "raw_scale": 1, "source_record": "claude-fable-5|loop_agent|mean-score", "source_model_id": "claude-fable-5", "n_samples": null, "reported_error": 4.9 }, { "id": "a9067d5c288020259124", "model": "Grok 4.6", "organization": "xAI", "configuration": "Agent: Loop; reasoning=xhigh; mean-score", "score": 60.3, "raw_score": 60.3, "raw_scale": 1, "source_record": "grok-4-6-xhigh|loop_agent|mean-score", "source_model_id": "grok-4-6-xhigh", "n_samples": null, "reported_error": 4.8 }, { "id": "943968ce488e0445eb7b", "model": "MiniMax-M3.1-Flash", "organization": "MiniMax", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 60.0, "raw_score": 60.0, "raw_scale": 1, "source_record": "minimax-m3p1-flash|loop_agent|mean-score", "source_model_id": "minimax-m3p1-flash", "n_samples": null, "reported_error": 4.9 }, { "id": "0d14188479a2fcdfd757", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 59.8, "raw_score": 59.8, "raw_scale": 1, "source_record": "gpt-5.6-terra-max|loop_agent|mean-score", "source_model_id": "gpt-5.6-terra-max", "n_samples": null, "reported_error": 4.8 }, { "id": "242a8075fb2b15cd17c5", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 59.7, "raw_score": 59.7, "raw_scale": 1, "source_record": "gpt-6-sol|loop_agent|mean-score", "source_model_id": "gpt-6-sol", "n_samples": null, "reported_error": 4.8 }, { "id": "6d9f562367c2cce4caa0", "model": "GLM-5.3", "organization": "Zhipu", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 59.5, "raw_score": 59.5, "raw_scale": 1, "source_record": "glm-5-3|loop_agent|mean-score", "source_model_id": "glm-5-3", "n_samples": null, "reported_error": 4.6 }, { "id": "ae97119105cc798f6324", "model": "DeepSeek-V4.1-Flash-1017", "organization": "DeepSeek", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 59.3, "raw_score": 59.3, "raw_scale": 1, "source_record": "deepseek-v4-1-flash-1017|loop_agent|mean-score", "source_model_id": "deepseek-v4-1-flash-1017", "n_samples": null, "reported_error": 5 }, { "id": "20b2aaceba3d849ec63f", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 58.8, "raw_score": 58.8, "raw_scale": 1, "source_record": "gemini-3.8-flash|loop_agent|mean-score", "source_model_id": "gemini-3.8-flash", "n_samples": null, "reported_error": 5.1 }, { "id": "82ec26e8efc291fea6c0", "model": "Muse Spark Open", "organization": "Meta", "configuration": "Agent: Loop; reasoning=xhigh; mean-score", "score": 58.0, "raw_score": 58.0, "raw_scale": 1, "source_record": "muse-spark-open|loop_agent|mean-score", "source_model_id": "muse-spark-open", "n_samples": null, "reported_error": 4.7 }, { "id": "0ad9f837327b64c2ce39", "model": "Grok 4.5", "organization": "xAI", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 57.4, "raw_score": 57.4, "raw_scale": 1, "source_record": "grok-4-5|loop_agent|mean-score", "source_model_id": "grok-4-5", "n_samples": null, "reported_error": 4.8 }, { "id": "6a3939c7a5e3638845a5", "model": "Gemini 3.7 Flash", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 57.2, "raw_score": 57.2, "raw_scale": 1, "source_record": "gemini-3.7-flash|loop_agent|mean-score", "source_model_id": "gemini-3.7-flash", "n_samples": null, "reported_error": 5.1 }, { "id": "1dccfc4683dff8c0ec86", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "Agent: Loop; reasoning=xhigh; mean-score", "score": 57.2, "raw_score": 57.2, "raw_scale": 1, "source_record": "qwen-3-8-max-xhigh|loop_agent|mean-score", "source_model_id": "qwen-3-8-max-xhigh", "n_samples": null, "reported_error": 5.1 }, { "id": "dd6668386fbd21d201e2", "model": "Kimi K3", "organization": "Kimi", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 56.7, "raw_score": 56.7, "raw_scale": 1, "source_record": "kimi-k3|loop_agent|mean-score", "source_model_id": "kimi-k3", "n_samples": null, "reported_error": 4.3 }, { "id": "06bbc7eadb03a6ef1f89", "model": "Mistral Large 4", "organization": "Mistral", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 56.5, "raw_score": 56.5, "raw_scale": 1, "source_record": "mistral-large-4|loop_agent|mean-score", "source_model_id": "mistral-large-4", "n_samples": null, "reported_error": 4.8 }, { "id": "2249f743701a7b02a182", "model": "Opus 4.7", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 56.5, "raw_score": 56.5, "raw_scale": 1, "source_record": "claude-opus-4-7|loop_agent|mean-score", "source_model_id": "claude-opus-4-7", "n_samples": null, "reported_error": 4.9 }, { "id": "2e872caa812a3248f16c", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=xhigh; mean-score", "score": 56.3, "raw_score": 56.3, "raw_scale": 1, "source_record": "gpt-5.5-xhigh|loop_agent|mean-score", "source_model_id": "gpt-5.5-xhigh", "n_samples": null, "reported_error": 4.9 }, { "id": "35edd667b87e758d9681", "model": "Sonnet 5", "organization": "Anthropic", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 55.7, "raw_score": 55.7, "raw_scale": 1, "source_record": "claude-sonnet-5-max|loop_agent|mean-score", "source_model_id": "claude-sonnet-5-max", "n_samples": null, "reported_error": 4.8 }, { "id": "4d4942d7cc2993f0a4f5", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: Loop; reasoning=xhigh; mean-score", "score": 55.3, "raw_score": 55.3, "raw_scale": 1, "source_record": "grok-4.7|loop_agent|mean-score", "source_model_id": "grok-4.7", "n_samples": null, "reported_error": 4.4 }, { "id": "36d69e2126644a4ef7ce", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 54.3, "raw_score": 54.3, "raw_scale": 1, "source_record": "gpt-5.6-luna-max|loop_agent|mean-score", "source_model_id": "gpt-5.6-luna-max", "n_samples": null, "reported_error": 5.1 }, { "id": "56c38e35bf712c050a69", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 53.9, "raw_score": 53.9, "raw_scale": 1, "source_record": "gpt-6-luna|loop_agent|mean-score", "source_model_id": "gpt-6-luna", "n_samples": null, "reported_error": 5.1 }, { "id": "9a04a7c40fb07dba6455", "model": "GLM-5.3-Flash", "organization": "Zhipu", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 53.7, "raw_score": 53.7, "raw_scale": 1, "source_record": "glm-5-3-flash|loop_agent|mean-score", "source_model_id": "glm-5-3-flash", "n_samples": null, "reported_error": 4.9 }, { "id": "fa8c8f49834a241fc712", "model": "DeepSeek-V4-Pro-0813", "organization": "DeepSeek", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 53.4, "raw_score": 53.4, "raw_scale": 1, "source_record": "deepseek-v4-pro-08-13|loop_agent|mean-score", "source_model_id": "deepseek-v4-pro-08-13", "n_samples": null, "reported_error": 4.6 }, { "id": "8a70ecd944c8a05e1650", "model": "DeepSeek-V4-Flash", "organization": "DeepSeek", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 53.3, "raw_score": 53.3, "raw_scale": 1, "source_record": "deepseek-v4-flash|loop_agent|mean-score", "source_model_id": "deepseek-v4-flash", "n_samples": null, "reported_error": 4.7 }, { "id": "1457df5d857bfcbc7001", "model": "Gemini 3.6 Flash", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 52.5, "raw_score": 52.5, "raw_scale": 1, "source_record": "gemini-3.6-flash|loop_agent|mean-score", "source_model_id": "gemini-3.6-flash", "n_samples": null, "reported_error": 4.7 }, { "id": "2529934359da12c58959", "model": "Mercury 3", "organization": "Inception", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 50.8, "raw_score": 50.8, "raw_scale": 1, "source_record": "inception/mercury-3|loop_agent|mean-score", "source_model_id": "inception/mercury-3", "n_samples": null, "reported_error": 4.4 }, { "id": "fcea3feeada3e4deeddf", "model": "DeepSeek-V4.1-Flash", "organization": "DeepSeek", "configuration": "Agent: Loop; reasoning=max; mean-score", "score": 50.1, "raw_score": 50.1, "raw_scale": 1, "source_record": "deepseek-v4-1-flash|loop_agent|mean-score", "source_model_id": "deepseek-v4-1-flash", "n_samples": null, "reported_error": 4.6 }, { "id": "d8d09e4c201697c19c95", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 41.8, "raw_score": 41.8, "raw_scale": 1, "source_record": "minimax-m3|loop_agent|mean-score", "source_model_id": "minimax-m3", "n_samples": null, "reported_error": 4.4 }, { "id": "19478626327f9fda4ec3", "model": "Nemotron 3.5 Super VL", "organization": "NVIDIA", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 38.1, "raw_score": 38.1, "raw_scale": 1, "source_record": "nemotron-3-5-super-vl|loop_agent|mean-score", "source_model_id": "nemotron-3-5-super-vl", "n_samples": null, "reported_error": 4.1 }, { "id": "ce982733296f839ae2a3", "model": "Nemotron 3.5 Bolt", "organization": "NVIDIA", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 35.7, "raw_score": 35.7, "raw_scale": 1, "source_record": "nemotron-3-5-bolt|loop_agent|mean-score", "source_model_id": "nemotron-3-5-bolt", "n_samples": null, "reported_error": 4.1 }, { "id": "f3664e43e4faf29c20b5", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 31.4, "raw_score": 31.4, "raw_scale": 1, "source_record": "gemini-3.5-flash|loop_agent|mean-score", "source_model_id": "gemini-3.5-flash", "n_samples": null, "reported_error": 4 }, { "id": "38851f1b9b47b555130d", "model": "Inkling", "organization": "Thinking Machines", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 31.2, "raw_score": 31.2, "raw_scale": 1, "source_record": "inkling|loop_agent|mean-score", "source_model_id": "inkling", "n_samples": null, "reported_error": 4 }, { "id": "7b03634adf9622c9d8f6", "model": "Nemotron 3 Ultra", "organization": "NVIDIA", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 29.4, "raw_score": 29.4, "raw_scale": 1, "source_record": "nemotron-3-ultra-nvfp4|loop_agent|mean-score", "source_model_id": "nemotron-3-ultra-nvfp4", "n_samples": null, "reported_error": 3.8 }, { "id": "9aa7ae65504f789d1810", "model": "Gemini 3.5 Flash Lite", "organization": "Google", "configuration": "Agent: Loop; reasoning=high; mean-score", "score": 5.2, "raw_score": 5.2, "raw_scale": 1, "source_record": "gemini-3-5-flash-lite|loop_agent|mean-score", "source_model_id": "gemini-3-5-flash-lite", "n_samples": null, "reported_error": 1.9 } ], "score_max": 100 }, { "id": "ace-mercor", "name": "AI Consumer Index (ACE)", "group": "专业工作", "unit": "score", "metric": "Mean rubric score", "protocol": { "en": "Mercor ACE v1, 400 held-out consumer tasks with web search. Mean hurdle- and grounding-adjusted rubric score on 0–100.", "zh": "Mercor ACE v1 的 400 道消费场景保留题，允许网络搜索。按目标达成与来源依据评分，取 0–100 量规分的平均值。" }, "source": { "name": "Mercor", "benchmark_creator": "Mercor", "url": "https://www.mercor.com/blog/introducing-the-ai-consumer-index/", "data_url": "https://www.mercor.com/apex/", "sha256": "b78316b4f963ace8a9365e6551de468728cb6180d19b961e1cdfc43e83f45ad6", "retrieved_at": "2026-11-02T05:58:01.181119+00:00", "selector": "benchmarkId=ace, globalLeaderboard.passScores[pass=pass-1].score; ACE v1 held-out mean rubric points", "kind": "official_leaderboard", "status": "current_snapshot" }, "results": [ { "id": "017f2933b97baeaf7e52", "model": "GPT-5", "organization": "OpenAI", "configuration": "reasoning=high; held-out mean rubric score", "score": 56.1, "raw_score": 56.1, "raw_scale": 1, "source_record": "gpt-5-high||pass-1", "source_model_id": "gpt-5-high", "n_samples": null, "reported_error": 3.3 }, { "id": "ae4e15f77e16477d1978", "model": "o3 Pro", "organization": "OpenAI", "configuration": "reasoning=high; held-out mean rubric score", "score": 55.2, "raw_score": 55.2, "raw_scale": 1, "source_record": "o3-pro||pass-1", "source_model_id": "o3-pro", "n_samples": null, "reported_error": 3.2 }, { "id": "fc931bf13d68d1e6532d", "model": "GPT-5.1", "organization": "OpenAI", "configuration": "reasoning=high; held-out mean rubric score", "score": 55.1, "raw_score": 55.1, "raw_scale": 1, "source_record": "gpt-5.1-high||pass-1", "source_model_id": "gpt-5.1-high", "n_samples": null, "reported_error": 3.2 }, { "id": "11d7fdcc0d7593ddf169", "model": "o3", "organization": "OpenAI", "configuration": "reasoning=high; held-out mean rubric score", "score": 52.9, "raw_score": 52.9, "raw_scale": 1, "source_record": "o3-high||pass-1", "source_model_id": "o3-high", "n_samples": null, "reported_error": 3.1 }, { "id": "31d9a935ba0217dcc3e5", "model": "GPT-5.2", "organization": "OpenAI", "configuration": "reasoning=high; held-out mean rubric score", "score": 51.5, "raw_score": 51.5, "raw_scale": 1, "source_record": "gpt-5.2-high||pass-1", "source_model_id": "gpt-5.2-high", "n_samples": null, "reported_error": 3.2 }, { "id": "657f6544fe6484519a62", "model": "Gemini 3 Pro", "organization": "Google", "configuration": "reasoning=high; held-out mean rubric score", "score": 45.7, "raw_score": 45.7, "raw_scale": 1, "source_record": "gemini-3-pro-preview-high||pass-1", "source_model_id": "gemini-3-pro-preview-high", "n_samples": null, "reported_error": 2.95 }, { "id": "8ca95e3219df146cd6c5", "model": "Opus 4.5", "organization": "Anthropic", "configuration": "reasoning=high; held-out mean rubric score", "score": 38.3, "raw_score": 38.3, "raw_scale": 1, "source_record": "claude-opus-4-5-high||pass-1", "source_model_id": "claude-opus-4-5-high", "n_samples": null, "reported_error": 3 }, { "id": "ec4d73b3f15d17a461c9", "model": "Gemini 3 Flash", "organization": "Google", "configuration": "reasoning=high; held-out mean rubric score", "score": 36.1, "raw_score": 36.1, "raw_scale": 1, "source_record": "gemini-3-flash-preview-high||pass-1", "source_model_id": "gemini-3-flash-preview-high", "n_samples": null, "reported_error": 3 }, { "id": "75d8dce3a52892c15e4a", "model": "Gemini 2.5 Flash (On)", "organization": "Google", "configuration": "held-out mean rubric score", "score": 35.7, "raw_score": 35.7, "raw_scale": 1, "source_record": "gemini-2.5-flash-high||pass-1", "source_model_id": "gemini-2.5-flash-high", "n_samples": null, "reported_error": 2.9 }, { "id": "f0cd8924db0123649faa", "model": "Sonnet 4.5", "organization": "Anthropic", "configuration": "held-out mean rubric score", "score": 35.5, "raw_score": 35.5, "raw_scale": 1, "source_record": "claude-sonnet-4.5||pass-1", "source_model_id": "claude-sonnet-4.5", "n_samples": null, "reported_error": 2.95 }, { "id": "29af4b0fd76c073d0fb7", "model": "Opus 4.1", "organization": "Anthropic", "configuration": "held-out mean rubric score", "score": 33.8, "raw_score": 33.8, "raw_scale": 1, "source_record": "claude-opus-4-1||pass-1", "source_model_id": "claude-opus-4-1", "n_samples": null, "reported_error": 2.8 }, { "id": "c112fbaf6aaab65b7429", "model": "Gemini 2.5 Pro (On)", "organization": "Google", "configuration": "held-out mean rubric score", "score": 31.9, "raw_score": 31.9, "raw_scale": 1, "source_record": "gemini-2.5-pro-high||pass-1", "source_model_id": "gemini-2.5-pro-high", "n_samples": null, "reported_error": 2.7 } ], "score_max": 100 }, { "id": "vending-bench-v1-history", "name": "Vending-Bench (2025)", "group": "预测与决策", "unit": "usd", "metric": "Mean final net worth (USD)", "protocol": { "en": "The publisher’s retired original Vending-Bench. Mean final net worth includes bank balance and inventory, in US dollars. API configurations remain separate.", "zh": "发布方已归档的原版 Vending-Bench。期末平均净资产包含银行余额与库存，以美元计。每个 API 配置分别保留。" }, "source": { "name": "Andon Labs", "benchmark_creator": "Andon Labs", "url": "https://andonlabs.com/evals/vending-bench", "data_url": "https://andonlabs.com/evals/vending-bench", "sha256": "e216074cffb6405008917265edbed908d7682e76aa1d5dcf41fb77d81b4a2274", "retrieved_at": "2026-11-02T06:06:52.792582+00:00", "selector": "Expanded original table data: Net worth (mean) column, in USD; exclude Human", "kind": "official_leaderboard", "status": "archived", "artifacts": [ { "data_url": "https://andonlabs.com/_app/immutable/nodes/25.B6tBZleY.js", "sha256": "3b0510a9bda2a62918ca7d7bd37db002e14241911e749d5ae57c71cc057abb75", "retrieved_at": "2026-11-02T06:08:29.618523+00:00" } ] }, "results": [ { "id": "0ca0629aa4ecc63d7e8f", "model": "Grok 4", "organization": "xAI", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 4694.15, "raw_score": 4694.15, "raw_scale": 1, "source_record": "Grok 4|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Grok 4" }, { "id": "f677baaebfabd2f295d9", "model": "Gemini 3 Pro", "organization": "Google", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 4387.93, "raw_score": 4387.93, "raw_scale": 1, "source_record": "Gemini 3 Pro|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Gemini 3 Pro" }, { "id": "be40ee39efbd55549040", "model": "GPT-5", "organization": "OpenAI", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 3578.9, "raw_score": 3578.9, "raw_scale": 1, "source_record": "GPT-5|Vending-Bench v1; mean final net worth; USD", "source_model_label": "GPT-5" }, { "id": "16792b9129666f00ee4d", "model": "Claude Sonnet 4.5", "organization": "Anthropic", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 2465.02, "raw_score": 2465.02, "raw_scale": 1, "source_record": "Claude Sonnet 4.5|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Claude Sonnet 4.5" }, { "id": "e060e755fc3a430a62ef", "model": "GPT-5.1", "organization": "OpenAI", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 2379.88, "raw_score": 2379.88, "raw_scale": 1, "source_record": "GPT-5.1|Vending-Bench v1; mean final net worth; USD", "source_model_label": "GPT-5.1" }, { "id": "19e6be5e7b0e91806bdf", "model": "Claude 3.5 Sonnet", "organization": "Anthropic", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 2217.93, "raw_score": 2217.93, "raw_scale": 1, "source_record": "Claude 3.5 Sonnet|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Claude 3.5 Sonnet" }, { "id": "1c344beb74cab29fb0e8", "model": "Claude Opus 4", "organization": "Anthropic", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 2077.41, "raw_score": 2077.41, "raw_scale": 1, "source_record": "Claude Opus 4|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Claude Opus 4" }, { "id": "4d55e8dfcf661842c177", "model": "o3", "organization": "OpenAI", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 1843.11, "raw_score": 1843.11, "raw_scale": 1, "source_record": "o3|Vending-Bench v1; mean final net worth; USD", "source_model_label": "o3" }, { "id": "d8918d0993e821e2ed22", "model": "Claude 3.7 Sonnet", "organization": "Anthropic", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 1567.9, "raw_score": 1567.9, "raw_scale": 1, "source_record": "Claude 3.7 Sonnet|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Claude 3.7 Sonnet" }, { "id": "83655d37dbb288dc182e", "model": "Kimi K2 Thinking", "organization": "Moonshot AI", "configuration": "Vending-Bench v1; mean final net worth; USD; source label=Kimi K2 Thinking (Moonshot API)", "score": 1295.8, "raw_score": 1295.8, "raw_scale": 1, "source_record": "Kimi K2 Thinking (Moonshot API)|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Kimi K2 Thinking (Moonshot API)" }, { "id": "ec0c00927155366c8b38", "model": "Gemini Robotics-ER 1.5", "organization": "Google", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 975.68, "raw_score": 975.68, "raw_scale": 1, "source_record": "Gemini Robotics-ER 1.5|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Gemini Robotics-ER 1.5" }, { "id": "9f277f8257aab7052798", "model": "Claude Sonnet 4", "organization": "Anthropic", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 968.31, "raw_score": 968.31, "raw_scale": 1, "source_record": "Claude Sonnet 4|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Claude Sonnet 4" }, { "id": "9089bc03413f8ff66822", "model": "o3-mini", "organization": "OpenAI", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 906.86, "raw_score": 906.86, "raw_scale": 1, "source_record": "o3-mini|Vending-Bench v1; mean final net worth; USD", "source_model_label": "o3-mini" }, { "id": "7febd3a5982f15c2cf36", "model": "Gemini 2.5 Pro (preview-03-25)", "organization": "Google", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 789.34, "raw_score": 789.34, "raw_scale": 1, "source_record": "Gemini 2.5 Pro (preview-03-25)|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Gemini 2.5 Pro (preview-03-25)" }, { "id": "6bbe29fed94b73828bc3", "model": "Kimi K2 Thinking", "organization": "Moonshot AI", "configuration": "Vending-Bench v1; mean final net worth; USD; source label=Kimi K2 Thinking (3rd party API)", "score": 648.93, "raw_score": 648.93, "raw_scale": 1, "source_record": "Kimi K2 Thinking (3rd party API)|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Kimi K2 Thinking (3rd party API)" }, { "id": "e8761966f70421c5cd43", "model": "Qwen3-235B-A22B", "organization": "Alibaba", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 615.52, "raw_score": 615.52, "raw_scale": 1, "source_record": "Qwen3-235B-A22B|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Qwen3-235B-A22B" }, { "id": "3c8ea3ccef00dd047118", "model": "gpt-oss-120b", "organization": "OpenAI", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 607.97, "raw_score": 607.97, "raw_scale": 1, "source_record": "gpt-oss-120b|Vending-Bench v1; mean final net worth; USD", "source_model_label": "gpt-oss-120b" }, { "id": "ac0f9b62f3ad25337623", "model": "Gemini 1.5 Pro", "organization": "Google", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 594.02, "raw_score": 594.02, "raw_scale": 1, "source_record": "Gemini 1.5 Pro|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Gemini 1.5 Pro" }, { "id": "35c40448b711d2f30551", "model": "GPT-4o mini", "organization": "OpenAI", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 582.33, "raw_score": 582.33, "raw_scale": 1, "source_record": "GPT-4o mini|Vending-Bench v1; mean final net worth; USD", "source_model_label": "GPT-4o mini" }, { "id": "ee860fc1ce2e6808bea4", "model": "GPT-5-mini", "organization": "OpenAI", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 580.77, "raw_score": 580.77, "raw_scale": 1, "source_record": "GPT-5-mini|Vending-Bench v1; mean final net worth; USD", "source_model_label": "GPT-5-mini" }, { "id": "96cc10e04dc488f715ff", "model": "Gemini 1.5 Flash", "organization": "Google", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 571.85, "raw_score": 571.85, "raw_scale": 1, "source_record": "Gemini 1.5 Flash|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Gemini 1.5 Flash" }, { "id": "c4560c6dfea6552f07f1", "model": "Claude 3.5 Haiku", "organization": "Anthropic", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 373.36, "raw_score": 373.36, "raw_scale": 1, "source_record": "Claude 3.5 Haiku|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Claude 3.5 Haiku" }, { "id": "9ddab846a8f930ac0e7d", "model": "Gemini 2.0 Flash", "organization": "Google", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 338.08, "raw_score": 338.08, "raw_scale": 1, "source_record": "Gemini 2.0 Flash|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Gemini 2.0 Flash" }, { "id": "f6b7eaf6fbf45a9dbdf0", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 335.46, "raw_score": 335.46, "raw_scale": 1, "source_record": "GPT-4o|Vending-Bench v1; mean final net worth; USD", "source_model_label": "GPT-4o" }, { "id": "c6add114e1d3263f2cdd", "model": "Gemini 2.0 Pro", "organization": "Google", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 273.7, "raw_score": 273.7, "raw_scale": 1, "source_record": "Gemini 2.0 Pro|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Gemini 2.0 Pro" }, { "id": "40cb9d9d4d8ca9913953", "model": "Llama 4 Maverick", "organization": "Meta", "configuration": "Vending-Bench v1; mean final net worth; USD", "score": 157.33, "raw_score": 157.33, "raw_scale": 1, "source_record": "Llama 4 Maverick|Vending-Bench v1; mean final net worth; USD", "source_model_label": "Llama 4 Maverick" } ], "public_history": true }, { "id": "tau3-banking-mercor", "name": "τ³-Banking (Mercor)", "group": "Agent 与工具", "unit": "%", "metric": "Pass@1", "protocol": { "en": "Mercor evaluation, τ² v1.0.1 task set, 97 tasks × 3 trials. Half-duplex agent, offline BM25 grep, GPT-5.2 low user simulator, Gemini 3.5 Flash judge. The 194-sample result is outside this complete 291-sample batch.", "zh": "Mercor 独立评测，τ² v1.0.1 题集，97 题各运行 3 次。半双工 Agent、离线 BM25 检索、GPT-5.2 low 用户模拟器与 Gemini 3.5 Flash 裁判。本批只纳入 291 个样本完整的成绩。" }, "source": { "name": "Mercor", "benchmark_creator": "Sierra / Princeton University", "url": "https://www.mercor.com/apex/partner-benchmarks/tau-3-banking-leaderboard/", "data_url": "https://www.mercor.com/apex/partner-benchmarks/tau-3-banking-leaderboard/", "sha256": "7f49339d307ae907d1457f03b5cf5efe266e84f09415508b68c44b258f34eca7", "retrieved_at": "2026-11-02T06:08:29.846233+00:00", "selector": "Partner leaderboard.rows; n_samples=291; score[pass-1]", "kind": "evaluator", "status": "current_snapshot", "artifacts": [ { "data_url": "https://www.mercor.com/apex/methodology/", "sha256": "4efca79d43944b2dc7dae6a4e6a19d282af66380ff11ab267590ee819eb586b4", "retrieved_at": "2026-11-02T06:12:25.248577+00:00" } ] }, "results": [ { "id": "f0bb7edc86af2251b77c", "model": "Kimi K3.1 Pro", "organization": "Kimi", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.615, "raw_score": 61.5, "raw_scale": 100, "source_record": "kimi-k3.1-pro", "source_model_id": "kimi-k3.1-pro", "n_samples": 291, "reported_error": 8.4 }, { "id": "fedcf0678c6ce4486fc3", "model": "Gemini 4 Argon", "organization": "Google", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.581, "raw_score": 58.1, "raw_scale": 100, "source_record": "gemini-4-argon", "source_model_id": "gemini-4-argon", "n_samples": 291, "reported_error": 8.9 }, { "id": "06dfe89ec35c251b2f55", "model": "Gemini 4 Carbon", "organization": "Google", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.557, "raw_score": 55.7, "raw_scale": 100, "source_record": "gemini-4-carbon", "source_model_id": "gemini-4-carbon", "n_samples": 291, "reported_error": 9.1 }, { "id": "005912bee6ee4543cec4", "model": "Opus 5.5", "organization": "Anthropic", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.55, "raw_score": 55.0, "raw_scale": 100, "source_record": "claude-opus-5-5", "source_model_id": "claude-opus-5-5", "n_samples": 291, "reported_error": 8.2 }, { "id": "7f4847a2f798a5848591", "model": "Fable 5.1", "organization": "Anthropic", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.54, "raw_score": 54.0, "raw_scale": 100, "source_record": "claude-fable-5.1", "source_model_id": "claude-fable-5.1", "n_samples": 291, "reported_error": 8.9 }, { "id": "c13e66682022f86247ee", "model": "Sonnet 5.5", "organization": "Anthropic", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.54, "raw_score": 54.0, "raw_scale": 100, "source_record": "claude-sonnet-5-5", "source_model_id": "claude-sonnet-5-5", "n_samples": 291, "reported_error": 8.9 }, { "id": "034a54b3f1215f7e6271", "model": "Muse Spark 1.3", "organization": "Meta", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=xhigh; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.522, "raw_score": 52.2, "raw_scale": 100, "source_record": "muse-spark-1-3", "source_model_id": "muse-spark-1-3", "n_samples": 291, "reported_error": 8.6 }, { "id": "89b974edf2623095f842", "model": "Muse Wonder 1.0", "organization": "Meta", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.519, "raw_score": 51.9, "raw_scale": 100, "source_record": "muse-wonder-1-0-max", "source_model_id": "muse-wonder-1-0-max", "n_samples": 291, "reported_error": 8.2 }, { "id": "319034f3e25d0d9bd6b3", "model": "Gemini 3.8 Flash", "organization": "Google", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.488, "raw_score": 48.8, "raw_scale": 100, "source_record": "gemini-3.8-flash", "source_model_id": "gemini-3.8-flash", "n_samples": 291, "reported_error": 8.8 }, { "id": "ab30e9c2085ea3cd6c86", "model": "GPT-6 Astra", "organization": "OpenAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.467, "raw_score": 46.7, "raw_scale": 100, "source_record": "gpt-6-astra-max", "source_model_id": "gpt-6-astra-max", "n_samples": 291, "reported_error": 9.1 }, { "id": "58ce9fbc3e4e0065ad0d", "model": "Qwen3.8-Max", "organization": "Alibaba", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=xhigh; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.467, "raw_score": 46.7, "raw_scale": 100, "source_record": "qwen-3-8-max-xhigh", "source_model_id": "qwen-3-8-max-xhigh", "n_samples": 291, "reported_error": 8.4 }, { "id": "3f9ebf8b911f42cb5415", "model": "Grok 4.6", "organization": "xAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=xhigh; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.457, "raw_score": 45.7, "raw_scale": 100, "source_record": "grok-4-6-xhigh", "source_model_id": "grok-4-6-xhigh", "n_samples": 291, "reported_error": 8.2 }, { "id": "f9ef6eedd766691f041b", "model": "Opus 5", "organization": "Anthropic", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.44, "raw_score": 44.0, "raw_scale": 100, "source_record": "claude-opus-5", "source_model_id": "claude-opus-5", "n_samples": 291, "reported_error": 8.1 }, { "id": "3b2aea01d81903732002", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.43, "raw_score": 43.0, "raw_scale": 100, "source_record": "gpt-5-6-sol-max-pro", "source_model_id": "gpt-5-6-sol-max-pro", "n_samples": 291, "reported_error": 8.2 }, { "id": "7388db16debdff683f90", "model": "GPT-6 Sol", "organization": "OpenAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.43, "raw_score": 43.0, "raw_scale": 100, "source_record": "gpt-6-sol", "source_model_id": "gpt-6-sol", "n_samples": 291, "reported_error": 8.6 }, { "id": "11474d221127f480d368", "model": "GPT-6.1 Sol", "organization": "OpenAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.43, "raw_score": 43.0, "raw_scale": 100, "source_record": "gpt-6-1-sol", "source_model_id": "gpt-6-1-sol", "n_samples": 291, "reported_error": 8.9 }, { "id": "26f6cf65acbd60862113", "model": "Haiku 5.5", "organization": "Anthropic", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.43, "raw_score": 43.0, "raw_scale": 100, "source_record": "claude-haiku-5-5", "source_model_id": "claude-haiku-5-5", "n_samples": 291, "reported_error": 8.6 }, { "id": "340615d0fd400bfe8731", "model": "DeepSeek-V4.1-Flash-1017", "organization": "DeepSeek", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.41600000000000004, "raw_score": 41.6, "raw_scale": 100, "source_record": "deepseek-v4-1-flash-1017", "source_model_id": "deepseek-v4-1-flash-1017", "n_samples": 291, "reported_error": 8.1 }, { "id": "02bf22bbad7f384bb395", "model": "GLM-5.3", "organization": "Zhipu", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "glm-5-3", "source_model_id": "glm-5-3", "n_samples": 291, "reported_error": 8.1 }, { "id": "eb146ee858963564141f", "model": "Mercury 3", "organization": "Inception", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.405, "raw_score": 40.5, "raw_scale": 100, "source_record": "inception/mercury-3", "source_model_id": "inception/mercury-3", "n_samples": 291, "reported_error": 8.1 }, { "id": "e595f278554e6e9750e2", "model": "DeepSeek-V4-Pro-0813", "organization": "DeepSeek", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.402, "raw_score": 40.2, "raw_scale": 100, "source_record": "deepseek-v4-pro-08-13", "source_model_id": "deepseek-v4-pro-08-13", "n_samples": 291, "reported_error": 8.1 }, { "id": "9a7e60cd1c7bb9d54938", "model": "GPT-6 Luna", "organization": "OpenAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.402, "raw_score": 40.2, "raw_scale": 100, "source_record": "gpt-6-luna", "source_model_id": "gpt-6-luna", "n_samples": 291, "reported_error": 8.2 }, { "id": "69de2a39a61a5be53e0e", "model": "GPT-5.6 Terra", "organization": "OpenAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.38799999999999996, "raw_score": 38.8, "raw_scale": 100, "source_record": "gpt-5.6-terra-max", "source_model_id": "gpt-5.6-terra-max", "n_samples": 291, "reported_error": 8.1 }, { "id": "99bf0ffb48ef969298a2", "model": "DeepSeek-V4.1-Flash", "organization": "DeepSeek", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "deepseek-v4-1-flash", "source_model_id": "deepseek-v4-1-flash", "n_samples": 291, "reported_error": 8.4 }, { "id": "7f72bacd8db1c175c8b2", "model": "GLM-5.3-Flash", "organization": "Zhipu", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.37799999999999995, "raw_score": 37.8, "raw_scale": 100, "source_record": "glm-5-3-flash", "source_model_id": "glm-5-3-flash", "n_samples": 291, "reported_error": 7.6 }, { "id": "c0e3c6c349da5c92acc4", "model": "GPT-6.1 Terra", "organization": "OpenAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.371, "raw_score": 37.1, "raw_scale": 100, "source_record": "gpt-6-1-terra", "source_model_id": "gpt-6-1-terra", "n_samples": 291, "reported_error": 8.2 }, { "id": "ba24d275ae233fc80d48", "model": "MiniMax-M3.1-Flash", "organization": "MiniMax", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.361, "raw_score": 36.1, "raw_scale": 100, "source_record": "minimax-m3p1-flash", "source_model_id": "minimax-m3p1-flash", "n_samples": 291, "reported_error": 7.9 }, { "id": "fbc0ae889c0c1ed26ffe", "model": "GPT-5.6 Luna", "organization": "OpenAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.35700000000000004, "raw_score": 35.7, "raw_scale": 100, "source_record": "gpt-5.6-luna-max", "source_model_id": "gpt-5.6-luna-max", "n_samples": 291, "reported_error": 7.9 }, { "id": "80a8bf700c991501c5d0", "model": "Kimi K3", "organization": "Kimi", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.34700000000000003, "raw_score": 34.7, "raw_scale": 100, "source_record": "kimi-k3", "source_model_id": "kimi-k3", "n_samples": 291, "reported_error": 7.6 }, { "id": "c58f2e5e1fb413077013", "model": "Beam", "organization": "Reflection", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=max; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.33, "raw_score": 33.0, "raw_scale": 100, "source_record": "beam-1-0-max", "source_model_id": "beam-1-0-max", "n_samples": 291, "reported_error": 8.1 }, { "id": "459415a966d88d3e30f7", "model": "Grok 4.7", "organization": "xAI", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=xhigh; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.326, "raw_score": 32.6, "raw_scale": 100, "source_record": "grok-4.7", "source_model_id": "grok-4.7", "n_samples": 291, "reported_error": 7.6 }, { "id": "13b76ebe52da50917179", "model": "MiniMax-M3", "organization": "MiniMax", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.21, "raw_score": 21.0, "raw_scale": 100, "source_record": "minimax-m3", "source_model_id": "minimax-m3", "n_samples": 291, "reported_error": 6.2 }, { "id": "b01f4122e1c1ac7bab55", "model": "Nemotron 3.5 Super VL", "organization": "NVIDIA", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.192, "raw_score": 19.2, "raw_scale": 100, "source_record": "nemotron-3-5-super-vl", "source_model_id": "nemotron-3-5-super-vl", "n_samples": 291, "reported_error": 6.4 }, { "id": "0d689b0331908e0fd18f", "model": "Gemini 3.5 Flash Lite", "organization": "Google", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.18600000000000003, "raw_score": 18.6, "raw_scale": 100, "source_record": "gemini-3-5-flash-lite", "source_model_id": "gemini-3-5-flash-lite", "n_samples": 291, "reported_error": 6 }, { "id": "ea65284c0eec774c0fe9", "model": "Nemotron 3 Ultra", "organization": "NVIDIA", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.18600000000000003, "raw_score": 18.6, "raw_scale": 100, "source_record": "nemotron-3-ultra-nvfp4", "source_model_id": "nemotron-3-ultra-nvfp4", "n_samples": 291, "reported_error": 6 }, { "id": "31100262fe7036cce8cc", "model": "Nemotron 3.5 Bolt", "organization": "NVIDIA", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.165, "raw_score": 16.5, "raw_scale": 100, "source_record": "nemotron-3-5-bolt", "source_model_id": "nemotron-3-5-bolt", "n_samples": 291, "reported_error": 6 }, { "id": "538c89ff59cdcfdf7e9f", "model": "Inkling", "organization": "Thinking Machines", "configuration": "Agent: Tau half-duplex; BM25 grep; reasoning=high; user simulator=GPT-5.2 low; judge=Gemini 3.5 Flash; 97 tasks × 3", "score": 0.155, "raw_score": 15.5, "raw_scale": 100, "source_record": "inkling", "source_model_id": "inkling", "n_samples": 291, "reported_error": 5.8 } ] }, { "id": "crmarena-original-paper", "name": "CRMArena (2025)", "group": "专业工作", "unit": "score", "metric": "Mean task score", "public_history": true, "protocol": { "en": "Historical 2025 results. Original CRMArena, paper v2 Table 2. The ALL column averages F1 for knowledge QA and exact match for the other eight tasks. Act, ReAct and function calling are preserved per result.", "zh": "2025 年历史成绩。原版 CRMArena 论文 v2 表 2。ALL 是九项任务的平均分，知识问答按 F1、另八项按完全匹配计分。各行保留 Act、ReAct 或函数调用配置。" }, "source": { "name": "Salesforce", "benchmark_creator": "Salesforce", "url": "https://arxiv.org/html/2411.02305v2#S3.T2", "data_url": "https://arxiv.org/html/2411.02305v2", "sha256": "2e9bb53b4d9e86c311b85295b4e8191f7559a6929f4e8c2f8f633baea22a9efa", "retrieved_at": "2026-11-02T05:58:29.399312+00:00", "selector": "Table 2, ALL column; nine tasks; each published agent/model pair", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "ee025f5519e4d4bc14e6", "model": "o1", "organization": "OpenAI", "configuration": "Agent: Function Calling; prompting", "score": 64.3, "raw_score": 64.3, "raw_scale": 1, "source_record": "Function Calling|o1 (prompt)", "source_model_label": "o1 (prompt)" }, { "id": "f15e7a10e555dd60fea0", "model": "o1", "organization": "OpenAI", "configuration": "Agent: ReAct", "score": 57.7, "raw_score": 57.7, "raw_scale": 1, "source_record": "ReAct|o1", "source_model_label": "o1" }, { "id": "01941074357fc6508523", "model": "gpt-4o", "organization": "OpenAI", "configuration": "Agent: Function Calling", "score": 54.4, "raw_score": 54.4, "raw_scale": 1, "source_record": "Function Calling|gpt-4o", "source_model_label": "gpt-4o" }, { "id": "4a8d7249c3f5a0f7f15b", "model": "llama3.1-405b", "organization": "Meta", "configuration": "Agent: Function Calling; prompting", "score": 51.3, "raw_score": 51.3, "raw_scale": 1, "source_record": "Function Calling|llama3.1-405b (prompt)", "source_model_label": "llama3.1-405b (prompt)" }, { "id": "adac3e60a7e738099026", "model": "claude-3.5-sonnet", "organization": "Anthropic", "configuration": "Agent: Function Calling", "score": 41.8, "raw_score": 41.8, "raw_scale": 1, "source_record": "Function Calling|claude-3.5-sonnet", "source_model_label": "claude-3.5-sonnet" }, { "id": "74742e55a62f967b5770", "model": "llama3.1-70b", "organization": "Meta", "configuration": "Agent: Function Calling; prompting", "score": 41.1, "raw_score": 41.1, "raw_scale": 1, "source_record": "Function Calling|llama3.1-70b (prompt)", "source_model_label": "llama3.1-70b (prompt)" }, { "id": "e105c735a26c9dc9d5aa", "model": "gpt-4o", "organization": "OpenAI", "configuration": "Agent: ReAct", "score": 38.2, "raw_score": 38.2, "raw_scale": 1, "source_record": "ReAct|gpt-4o", "source_model_label": "gpt-4o" }, { "id": "3abd5aa333e49d5f1e18", "model": "claude-3.5-sonnet", "organization": "Anthropic", "configuration": "Agent: Act", "score": 37.4, "raw_score": 37.4, "raw_scale": 1, "source_record": "Act|claude-3.5-sonnet", "source_model_label": "claude-3.5-sonnet" }, { "id": "f1ad3484d5dfc048e8b9", "model": "deepseek-r1", "organization": "DeepSeek", "configuration": "Agent: ReAct", "score": 35.1, "raw_score": 35.1, "raw_scale": 1, "source_record": "ReAct|deepseek-r1", "source_model_label": "deepseek-r1" }, { "id": "8ec65b5734629969c4e5", "model": "claude-3.5-sonnet", "organization": "Anthropic", "configuration": "Agent: ReAct", "score": 34.3, "raw_score": 34.3, "raw_scale": 1, "source_record": "ReAct|claude-3.5-sonnet", "source_model_label": "claude-3.5-sonnet" }, { "id": "e9bd83826259fc9e1647", "model": "llama3.1-405b", "organization": "Meta", "configuration": "Agent: ReAct", "score": 33.8, "raw_score": 33.8, "raw_scale": 1, "source_record": "ReAct|llama3.1-405b", "source_model_label": "llama3.1-405b" }, { "id": "9a24f01e7df85a2c6dee", "model": "gpt-4o", "organization": "OpenAI", "configuration": "Agent: Act", "score": 29.4, "raw_score": 29.4, "raw_scale": 1, "source_record": "Act|gpt-4o", "source_model_label": "gpt-4o" }, { "id": "d5ed859fb2078e01a166", "model": "gpt-4o-mini", "organization": "OpenAI", "configuration": "Agent: ReAct", "score": 28.3, "raw_score": 28.3, "raw_scale": 1, "source_record": "ReAct|gpt-4o-mini", "source_model_label": "gpt-4o-mini" }, { "id": "5ea9493277888138f70b", "model": "llama3.1-70b", "organization": "Meta", "configuration": "Agent: ReAct", "score": 27.8, "raw_score": 27.8, "raw_scale": 1, "source_record": "ReAct|llama3.1-70b", "source_model_label": "llama3.1-70b" }, { "id": "850122bbf6eed606ea63", "model": "llama3.1-405b", "organization": "Meta", "configuration": "Agent: Act", "score": 22.2, "raw_score": 22.2, "raw_scale": 1, "source_record": "Act|llama3.1-405b", "source_model_label": "llama3.1-405b" }, { "id": "c5ceeeebb9f9f3782fec", "model": "gpt-4o-mini", "organization": "OpenAI", "configuration": "Agent: Function Calling", "score": 19.5, "raw_score": 19.5, "raw_scale": 1, "source_record": "Function Calling|gpt-4o-mini", "source_model_label": "gpt-4o-mini" }, { "id": "37fd5fd7125ba88302ea", "model": "llama3.1-70b", "organization": "Meta", "configuration": "Agent: Act", "score": 18.6, "raw_score": 18.6, "raw_scale": 1, "source_record": "Act|llama3.1-70b", "source_model_label": "llama3.1-70b" }, { "id": "061c88f85474313c5bf3", "model": "claude-3-sonnet", "organization": "Anthropic", "configuration": "Agent: ReAct", "score": 17.3, "raw_score": 17.3, "raw_scale": 1, "source_record": "ReAct|claude-3-sonnet", "source_model_label": "claude-3-sonnet" }, { "id": "234b046a4c447f5bc2aa", "model": "gpt-4o-mini", "organization": "OpenAI", "configuration": "Agent: Act", "score": 16.7, "raw_score": 16.7, "raw_scale": 1, "source_record": "Act|gpt-4o-mini", "source_model_label": "gpt-4o-mini" }, { "id": "c012e3e507dc65303024", "model": "claude-3-sonnet", "organization": "Anthropic", "configuration": "Agent: Act", "score": 16.6, "raw_score": 16.6, "raw_scale": 1, "source_record": "Act|claude-3-sonnet", "source_model_label": "claude-3-sonnet" }, { "id": "8135c47594573e133db4", "model": "claude-3-sonnet", "organization": "Anthropic", "configuration": "Agent: Function Calling", "score": 15.1, "raw_score": 15.1, "raw_scale": 1, "source_record": "Function Calling|claude-3-sonnet", "source_model_label": "claude-3-sonnet" }, { "id": "1c00253df915751de7f3", "model": "deepseek-r1", "organization": "DeepSeek", "configuration": "Agent: Function Calling; prompting", "score": 9.0, "raw_score": 9.0, "raw_scale": 1, "source_record": "Function Calling|deepseek-r1 (prompt)", "source_model_label": "deepseek-r1 (prompt)" }, { "id": "3a16e7dd814b23c11c62", "model": "llama3.1-8b", "organization": "Meta", "configuration": "Agent: ReAct", "score": 3.1, "raw_score": 3.1, "raw_scale": 1, "source_record": "ReAct|llama3.1-8b", "source_model_label": "llama3.1-8b" }, { "id": "6bf1e015cf88c0cbef70", "model": "llama3.1-8b", "organization": "Meta", "configuration": "Agent: Act", "score": 2.5, "raw_score": 2.5, "raw_scale": 1, "source_record": "Act|llama3.1-8b", "source_model_label": "llama3.1-8b" }, { "id": "a25aaf3796c87fefb7b5", "model": "llama3.1-8b", "organization": "Meta", "configuration": "Agent: Function Calling; prompting", "score": 0.0, "raw_score": 0.0, "raw_scale": 1, "source_record": "Function Calling|llama3.1-8b (prompt)", "source_model_label": "llama3.1-8b (prompt)" } ] }, { "id": "google-facts-2025-paper", "name": "FACTS Benchmark Suite (2025)", "group": "知识与推理", "unit": "score", "metric": "FACTS composite score", "public_history": true, "protocol": { "en": "Historical 2025 results. December 2025 paper Table 1. Published four-component average with Grounding v2 and the paper versions of Multimodal, Parametric and Search. The original 0–100 points are retained.", "zh": "2025 年历史成绩。2025 年 12 月论文表 1。原表四项平均分使用 Grounding v2，以及论文当时的 Multimodal、Parametric、Search 版本。保留原表 0–100 综合分。" }, "source": { "name": "Google DeepMind / Google Research", "benchmark_creator": "Google DeepMind / Google Research", "url": "https://arxiv.org/html/2512.10791v1#S1.T1", "data_url": "https://arxiv.org/html/2512.10791v1", "sha256": "d39b1a0d99d5695e7b336edebce0432a0a18e07b56cdf66872ca0820f6fee80d", "retrieved_at": "2026-11-02T05:58:22.889955+00:00", "selector": "Paper Table 1: FACTS Score, 15 named models", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "c26d808336a41220f3e7", "model": "Gemini 3 Pro", "organization": "Google", "configuration": "Table 1; four-component average; Grounding v2", "score": 68.8, "raw_score": 68.8, "raw_scale": 1, "source_record": "S1.T1.6.2", "source_date": "2025-12-11" }, { "id": "a1ee664df4d6a8d9ee9e", "model": "Gemini 2.5 Pro", "organization": "Google", "configuration": "Table 1; four-component average; Grounding v2", "score": 62.1, "raw_score": 62.1, "raw_scale": 1, "source_record": "S1.T1.6.3", "source_date": "2025-12-11" }, { "id": "a529bbc41ea078bfa21e", "model": "GPT 5", "organization": "OpenAI", "configuration": "Table 1; four-component average; Grounding v2", "score": 61.8, "raw_score": 61.8, "raw_scale": 1, "source_record": "S1.T1.6.4", "source_date": "2025-12-11" }, { "id": "d3110c54a99c61e74f2e", "model": "Grok 4", "organization": "xAI", "configuration": "Table 1; four-component average; Grounding v2", "score": 53.6, "raw_score": 53.6, "raw_scale": 1, "source_record": "S1.T1.6.5", "source_date": "2025-12-11" }, { "id": "afff1a87f6189e8a668a", "model": "GPT o3", "organization": "OpenAI", "configuration": "Table 1; four-component average; Grounding v2", "score": 52.0, "raw_score": 52.0, "raw_scale": 1, "source_record": "S1.T1.6.6", "source_date": "2025-12-11" }, { "id": "03bd314ac1a0210235aa", "model": "Claude 4.5 Opus", "organization": "Anthropic", "configuration": "Table 1; four-component average; Grounding v2", "score": 51.3, "raw_score": 51.3, "raw_scale": 1, "source_record": "S1.T1.6.7", "source_date": "2025-12-11" }, { "id": "a69cb2203439207c5383", "model": "GPT 4.1", "organization": "OpenAI", "configuration": "Table 1; four-component average; Grounding v2", "score": 50.5, "raw_score": 50.5, "raw_scale": 1, "source_record": "S1.T1.6.8", "source_date": "2025-12-11" }, { "id": "ab940c2ccd9a2de53311", "model": "Gemini 2.5 Flash", "organization": "Google", "configuration": "Table 1; four-component average; Grounding v2", "score": 50.4, "raw_score": 50.4, "raw_scale": 1, "source_record": "S1.T1.6.9", "source_date": "2025-12-11" }, { "id": "d4e7bd716d04d0d77e18", "model": "GPT 5.1", "organization": "OpenAI", "configuration": "Table 1; four-component average; Grounding v2", "score": 49.4, "raw_score": 49.4, "raw_scale": 1, "source_record": "S1.T1.6.10", "source_date": "2025-12-11" }, { "id": "ace3ed8bd007d883332b", "model": "Claude 4.5 Sonnet Thinking", "organization": "Anthropic", "configuration": "Table 1; four-component average; Grounding v2", "score": 49.1, "raw_score": 49.1, "raw_scale": 1, "source_record": "S1.T1.6.11", "source_date": "2025-12-11" }, { "id": "f68dae11fa0711dac02a", "model": "Claude 4.1 Opus", "organization": "Anthropic", "configuration": "Table 1; four-component average; Grounding v2", "score": 46.5, "raw_score": 46.5, "raw_scale": 1, "source_record": "S1.T1.6.12", "source_date": "2025-12-11" }, { "id": "924f961b45b412de607f", "model": "GPT 5 mini", "organization": "OpenAI", "configuration": "Table 1; four-component average; Grounding v2", "score": 45.9, "raw_score": 45.9, "raw_scale": 1, "source_record": "S1.T1.6.13", "source_date": "2025-12-11" }, { "id": "75e2dbe4952a3a52077b", "model": "Claude 4 Sonnet", "organization": "Anthropic", "configuration": "Table 1; four-component average; Grounding v2", "score": 42.8, "raw_score": 42.8, "raw_scale": 1, "source_record": "S1.T1.6.14", "source_date": "2025-12-11" }, { "id": "6740c4d7ab59c056d7af", "model": "GPT o4 mini", "organization": "OpenAI", "configuration": "Table 1; four-component average; Grounding v2", "score": 37.6, "raw_score": 37.6, "raw_scale": 1, "source_record": "S1.T1.6.15", "source_date": "2025-12-11" }, { "id": "77b24f22666bf1e39804", "model": "Grok 4 Fast", "organization": "xAI", "configuration": "Table 1; four-component average; Grounding v2", "score": 36.0, "raw_score": 36.0, "raw_scale": 1, "source_record": "S1.T1.6.16", "source_date": "2025-12-11" } ] }, { "id": "scicode-authors-main", "name": "SciCode (authors’ leaderboard, main problems) (2024–2025)", "group": "编程", "unit": "%", "metric": "Main problem resolve rate", "public_history": true, "protocol": { "en": "Historical 2024–2025 results. Original SciCode authors’ published main-problem leaderboard. Each reasoning setting keeps its source label. The separate subproblem column is omitted.", "zh": "2024–2025 年历史成绩。SciCode 原作者榜单的完整问题解决率。各推理设置沿用原榜标签，分步子题另列，未计入本项。" }, "source": { "name": "SciCode authors", "benchmark_creator": "SciCode authors", "url": "https://scicode-bench.github.io/leaderboard/", "data_url": "https://raw.githubusercontent.com/scicode-bench/SciCode/e3158ea011d4235245a547460d3688d7ccbf9900/README.md", "sha256": "a06846f318d4d269449d83201169058bc4e1ee64f3375a6104c4477d6ef03799", "retrieved_at": "2026-11-02T06:03:22.332789+00:00", "selector": "Pinned author README leaderboard: Main Problem Resolve Rate", "kind": "official_leaderboard", "status": "dated_release" }, "results": [ { "id": "936cd3efba8369c8137f", "model": "OpenAI o3-mini-low", "organization": "OpenAI", "configuration": "Original author leaderboard; main problems", "score": 0.10800000000000001, "raw_score": 10.8, "raw_scale": 100, "source_record": "OpenAI o3-mini-low|Original author leaderboard; main problems" }, { "id": "f27699fcd43430ee4778", "model": "OpenAI o3-mini-high", "organization": "OpenAI", "configuration": "Original author leaderboard; main problems", "score": 0.092, "raw_score": 9.2, "raw_scale": 100, "source_record": "OpenAI o3-mini-high|Original author leaderboard; main problems" }, { "id": "b160ebd2ed44bcedd779", "model": "OpenAI o3-mini-medium", "organization": "OpenAI", "configuration": "Original author leaderboard; main problems", "score": 0.092, "raw_score": 9.2, "raw_scale": 100, "source_record": "OpenAI o3-mini-medium|Original author leaderboard; main problems" }, { "id": "410320eef39ddb7b9d6e", "model": "OpenAI o1-preview", "organization": "OpenAI", "configuration": "Original author leaderboard; main problems", "score": 0.077, "raw_score": 7.7, "raw_scale": 100, "source_record": "OpenAI o1-preview|Original author leaderboard; main problems" }, { "id": "85fd5550b22928421479", "model": "Claude3.5-Sonnet", "organization": "Anthropic", "configuration": "Original author leaderboard; main problems", "score": 0.046, "raw_score": 4.6, "raw_scale": 100, "source_record": "Claude3.5-Sonnet|Original author leaderboard; main problems" }, { "id": "475460156fcfac5bf50e", "model": "Claude3.5-Sonnet (new)", "organization": "Anthropic", "configuration": "Original author leaderboard; main problems", "score": 0.046, "raw_score": 4.6, "raw_scale": 100, "source_record": "Claude3.5-Sonnet (new)|Original author leaderboard; main problems" }, { "id": "475cd26f2c7c4834b61d", "model": "Deepseek-R1", "organization": "DeepSeek", "configuration": "Original author leaderboard; main problems", "score": 0.046, "raw_score": 4.6, "raw_scale": 100, "source_record": "Deepseek-R1|Original author leaderboard; main problems" }, { "id": "e28cdbb4039db65cd0d6", "model": "Deepseek-Coder-v2", "organization": "DeepSeek", "configuration": "Original author leaderboard; main problems", "score": 0.031, "raw_score": 3.1, "raw_scale": 100, "source_record": "Deepseek-Coder-v2|Original author leaderboard; main problems" }, { "id": "660e8ad9a28c10f8c944", "model": "Deepseek-v3", "organization": "DeepSeek", "configuration": "Original author leaderboard; main problems", "score": 0.031, "raw_score": 3.1, "raw_scale": 100, "source_record": "Deepseek-v3|Original author leaderboard; main problems" }, { "id": "f23f1af86e39296a693b", "model": "Claude3-Opus", "organization": "Anthropic", "configuration": "Original author leaderboard; main problems", "score": 0.015, "raw_score": 1.5, "raw_scale": 100, "source_record": "Claude3-Opus|Original author leaderboard; main problems" }, { "id": "6791e99968aa85e41685", "model": "Claude3-Sonnet", "organization": "Anthropic", "configuration": "Original author leaderboard; main problems", "score": 0.015, "raw_score": 1.5, "raw_scale": 100, "source_record": "Claude3-Sonnet|Original author leaderboard; main problems" }, { "id": "841f57a27b99d91c5c0e", "model": "GPT-4-Turbo", "organization": "OpenAI", "configuration": "Original author leaderboard; main problems", "score": 0.015, "raw_score": 1.5, "raw_scale": 100, "source_record": "GPT-4-Turbo|Original author leaderboard; main problems" }, { "id": "ec220d756b73c78bf418", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Original author leaderboard; main problems", "score": 0.015, "raw_score": 1.5, "raw_scale": 100, "source_record": "GPT-4o|Original author leaderboard; main problems" }, { "id": "8fdb44091aa98c5421ec", "model": "Gemini 1.5 Pro", "organization": "Google", "configuration": "Original author leaderboard; main problems", "score": 0.015, "raw_score": 1.5, "raw_scale": 100, "source_record": "Gemini 1.5 Pro|Original author leaderboard; main problems" }, { "id": "4a1f2fcfcaa58acbde57", "model": "Llama-3.1-405B-Chat", "organization": "Meta", "configuration": "Original author leaderboard; main problems", "score": 0.015, "raw_score": 1.5, "raw_scale": 100, "source_record": "Llama-3.1-405B-Chat|Original author leaderboard; main problems" }, { "id": "f5867d37e5fe9f865643", "model": "OpenAI o1-mini", "organization": "OpenAI", "configuration": "Original author leaderboard; main problems", "score": 0.015, "raw_score": 1.5, "raw_scale": 100, "source_record": "OpenAI o1-mini|Original author leaderboard; main problems" }, { "id": "3312de55db836c6bba1a", "model": "Qwen2-72B-Instruct", "organization": "Alibaba", "configuration": "Original author leaderboard; main problems", "score": 0.015, "raw_score": 1.5, "raw_scale": 100, "source_record": "Qwen2-72B-Instruct|Original author leaderboard; main problems" }, { "id": "c551c5833e2d421f1341", "model": "Llama-3-70B-Chat", "organization": "Meta", "configuration": "Original author leaderboard; main problems", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "Llama-3-70B-Chat|Original author leaderboard; main problems" }, { "id": "8bc31e52bfa97fd72968", "model": "Llama-3.1-70B-Chat", "organization": "Meta", "configuration": "Original author leaderboard; main problems", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "Llama-3.1-70B-Chat|Original author leaderboard; main problems" }, { "id": "4031a3906bf8aa4f1f51", "model": "Mixtral-8x22B-Instruct", "organization": "Mistral AI", "configuration": "Original author leaderboard; main problems", "score": 0.0, "raw_score": 0.0, "raw_scale": 100, "source_record": "Mixtral-8x22B-Instruct|Original author leaderboard; main problems" } ] }, { "id": "scicode-original-matched-paper", "name": "SciCode (matched original rerun) (2026-08)", "group": "编程", "unit": "%", "metric": "Main problem pass@1", "public_history": true, "protocol": { "en": "Historical 2026-08 results. Paper Table 1, 64 whole problems with background, pass@1 and two-environment OR grading. The authors reran the original test data under the matched protocol.", "zh": "2026-08 年历史成绩。论文表 1，64 道完整题，提供背景，pass@1；两个环境中有一个通过即算通过。作者按相同协议重新运行原始题集。" }, "source": { "name": "SciCode-Verified authors", "benchmark_creator": "SciCode-Verified authors", "url": "https://arxiv.org/html/2608.04975v1#S3.T1", "data_url": "https://arxiv.org/html/2608.04975v1", "sha256": "a6d9f5967dcfc18439cbc35b62631d0ecbe4bf6848a34d04729f44e460b9ceb4", "retrieved_at": "2026-11-02T06:03:23.254617+00:00", "selector": "Table 1: Original main %, matched rerun", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "b7dd29ff556261c839d4", "model": "Seed 2.1 Pro", "organization": "", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.266, "raw_score": 26.6, "raw_scale": 100, "source_record": "Seed 2.1 Pro|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "de42c1c8a3e439ba3596", "model": "Claude Opus 4.8", "organization": "Anthropic", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.25, "raw_score": 25.0, "raw_scale": 100, "source_record": "Claude Opus 4.8|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "2d69eab03a4cbc1c04e1", "model": "Claude Fable 5", "organization": "Anthropic", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.23399999999999999, "raw_score": 23.4, "raw_scale": 100, "source_record": "Claude Fable 5|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "563e3163a61e3a2bb4c7", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.23399999999999999, "raw_score": 23.4, "raw_scale": 100, "source_record": "GPT-5.6 Sol|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "23879545efc2afa82c18", "model": "Muse Spark 1.1", "organization": "Meta", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.23399999999999999, "raw_score": 23.4, "raw_scale": 100, "source_record": "Muse Spark 1.1|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "1a4ff2aab2a6a159dcff", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.21899999999999997, "raw_score": 21.9, "raw_scale": 100, "source_record": "Kimi K3|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "a6132daff071af5730a8", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.203, "raw_score": 20.3, "raw_scale": 100, "source_record": "GPT-5.5|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "2b3c75fbe87269030420", "model": "DeepSeek V4 Flash", "organization": "DeepSeek", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.188, "raw_score": 18.8, "raw_scale": 100, "source_record": "DeepSeek V4 Flash|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "e67af6bf821d5142ebd8", "model": "DeepSeek V4 Pro Preview", "organization": "DeepSeek", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.172, "raw_score": 17.2, "raw_scale": 100, "source_record": "DeepSeek V4 Pro Preview|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "cb1ec9cd32e04665fe6b", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.172, "raw_score": 17.2, "raw_scale": 100, "source_record": "Gemini 3.5 Flash|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "e177291c02f3e0657e3c", "model": "GLM-5.2", "organization": "Z AI", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.156, "raw_score": 15.6, "raw_scale": 100, "source_record": "GLM-5.2|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "550a3903576286d46d7a", "model": "DeepSeek V4 Flash Preview", "organization": "DeepSeek", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.094, "raw_score": 9.4, "raw_scale": 100, "source_record": "DeepSeek V4 Flash Preview|64 main problems; background provided; pass@1; two-environment OR grading" } ] }, { "id": "scicode-verified-main-paper", "name": "SciCode-Verified (2026-08)", "group": "编程", "unit": "%", "metric": "Main problem pass@1", "public_history": true, "protocol": { "en": "Historical 2026-08 results. Paper Table 1, 64 whole problems with background, pass@1 and two-environment OR grading. Corrected test data, main harness; two starred earlier-harness means are excluded.", "zh": "2026-08 年历史成绩。论文表 1，64 道完整题，提供背景，pass@1；两个环境中有一个通过即算通过。修订题集及主评测框架，排除两项标星的旧框架多次运行均值。" }, "source": { "name": "SciCode-Verified authors", "benchmark_creator": "SciCode-Verified authors", "url": "https://arxiv.org/html/2608.04975v1#S3.T1", "data_url": "https://arxiv.org/html/2608.04975v1", "sha256": "a6d9f5967dcfc18439cbc35b62631d0ecbe4bf6848a34d04729f44e460b9ceb4", "retrieved_at": "2026-11-02T06:03:23.254617+00:00", "selector": "Table 1: SciCode-Verified main %, excluding earlier-harness starred rows", "kind": "official_leaderboard", "status": "paper_baselines" }, "results": [ { "id": "563e3163a61e3a2bb4c7", "model": "GPT-5.6 Sol", "organization": "OpenAI", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.922, "raw_score": 92.2, "raw_scale": 100, "source_record": "GPT-5.6 Sol|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "a6132daff071af5730a8", "model": "GPT-5.5", "organization": "OpenAI", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.9059999999999999, "raw_score": 90.6, "raw_scale": 100, "source_record": "GPT-5.5|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "cb1ec9cd32e04665fe6b", "model": "Gemini 3.5 Flash", "organization": "Google", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.875, "raw_score": 87.5, "raw_scale": 100, "source_record": "Gemini 3.5 Flash|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "2b3c75fbe87269030420", "model": "DeepSeek V4 Flash", "organization": "DeepSeek", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.8440000000000001, "raw_score": 84.4, "raw_scale": 100, "source_record": "DeepSeek V4 Flash|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "2d69eab03a4cbc1c04e1", "model": "Claude Fable 5", "organization": "Anthropic", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.812, "raw_score": 81.2, "raw_scale": 100, "source_record": "Claude Fable 5|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "b7dd29ff556261c839d4", "model": "Seed 2.1 Pro", "organization": "", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.797, "raw_score": 79.7, "raw_scale": 100, "source_record": "Seed 2.1 Pro|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "de42c1c8a3e439ba3596", "model": "Claude Opus 4.8", "organization": "Anthropic", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.7809999999999999, "raw_score": 78.1, "raw_scale": 100, "source_record": "Claude Opus 4.8|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "1a4ff2aab2a6a159dcff", "model": "Kimi K3", "organization": "Moonshot AI", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.7659999999999999, "raw_score": 76.6, "raw_scale": 100, "source_record": "Kimi K3|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "23879545efc2afa82c18", "model": "Muse Spark 1.1", "organization": "Meta", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.7659999999999999, "raw_score": 76.6, "raw_scale": 100, "source_record": "Muse Spark 1.1|64 main problems; background provided; pass@1; two-environment OR grading" }, { "id": "e177291c02f3e0657e3c", "model": "GLM-5.2", "organization": "Z AI", "configuration": "64 main problems; background provided; pass@1; two-environment OR grading", "score": 0.703, "raw_score": 70.3, "raw_scale": 100, "source_record": "GLM-5.2|64 main problems; background provided; pass@1; two-environment OR grading" } ] }, { "id": "browsecomp-kaggle-v1", "name": "BrowseComp (Kaggle v1)", "group": "Agent 与工具", "unit": "%", "metric": "Score", "protocol": { "en": "Kaggle’s OpenAI collection, BrowseComp v1. The Score task reports one attempt and selects the latest run for each model version. The leaderboard does not specify a browsing tool or agent.", "zh": "Kaggle OpenAI 名下的 BrowseComp v1。Score 任务记录每个模型版本的一次尝试，采用最近一次运行。榜单未标明浏览工具或智能体。" }, "source": { "data_url": "https://www.kaggle.com/api/v1/benchmarks/openai/browsecomp/leaderboard?version=1", "sha256": "207711c7e013869cdc931f84a9f4eda4b5906f77da734d701ccb04f411f1a82f", "retrieved_at": "2026-11-02T06:13:30.403188+00:00", "name": "Kaggle Benchmarks / OpenAI collection", "url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "kind": "evaluator", "status": "current_snapshot", "benchmark_creator": "OpenAI", "score_publisher": "Kaggle Benchmarks", "selector": "Public v1 leaderboard, modelVersion candidates, Score task v1, numericResult.value, native proportion", "published_at": "2025-07-15T14:25:30.827974100Z", "version_updated_at": "2026-03-26T07:00:41.089714400Z", "artifacts": [ { "data_url": "https://www.kaggle.com/api/i/benchmarks.BenchmarkService/GetBenchmark", "sha256": "a83898cea7e9496470afc63a7f8b17206f1f7151221f7479ca5cbc480d2e6464", "retrieved_at": "2026-11-02T06:14:41.994928+00:00", "method": "POST", "request": { "benchmarkIdentifier": { "slugIdentifier": { "ownerSlug": "openai", "benchmarkSlug": "browsecomp" } }, "versionIdentifier": { "versionNumberSelector": { "parentBenchmarkIdentifier": { "slugIdentifier": { "ownerSlug": "openai", "benchmarkSlug": "browsecomp" } }, "versionNumber": 1 } } } } ] }, "results": [ { "id": "777b26eaaf39ff7b2a37", "model": "Gemini 3.1 Pro Preview", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemini-3.1-pro-preview", "score": 0.3118536197295147, "raw_score": 0.3118536197295147, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemini-3.1-pro-preview", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemini-3.1-pro-preview", "source_model_name": "Gemini 3.1 Pro Preview", "reported_confidence_interval": 0.0256092147347973 }, { "id": "e60be172c3335a8d946e", "model": "GPT-5", "organization": "OpenAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gpt-5-2025-08-07", "score": 0.2006319115323854, "raw_score": 0.2006319115323854, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gpt-5-2025-08-07", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gpt-5-2025-08-07", "source_model_name": "GPT-5", "reported_confidence_interval": 0.0220599562632808, "source_date": "2025-08-26T00:00:00Z" }, { "id": "71e28e3ed65681ff0ab8", "model": "Gemini 3 Flash Preview", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemini-3-flash-preview", "score": 0.1691699604743083, "raw_score": 0.1691699604743083, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemini-3-flash-preview", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemini-3-flash-preview", "source_model_name": "Gemini 3 Flash Preview", "reported_confidence_interval": 0.0206595352448205 }, { "id": "c77e7be1e51c6c3405b0", "model": "Gemini 2.5 Pro Preview", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemini-2.5-pro-preview-06-05", "score": 0.0781990521327014, "raw_score": 0.0781990521327014, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemini-2.5-pro-preview-06-05", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemini-2.5-pro-preview-06-05", "source_model_name": "Gemini 2.5 Pro Preview", "reported_confidence_interval": 0.0147894000891353, "source_date": "2025-06-13T00:00:00Z" }, { "id": "3d484ac1eefeaeef8862", "model": "o1", "organization": "OpenAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version o1-2024-12-17", "score": 0.0631911532385466, "raw_score": 0.0631911532385466, "raw_scale": 1, "source_record": "browsecomp|v1|Score|o1-2024-12-17", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "o1-2024-12-17", "source_model_name": "o1", "reported_confidence_interval": 0.0134024708249485, "source_date": "2025-06-13T00:00:00Z" }, { "id": "c06bcd4beab51d37e6ea", "model": "GPT-4.1", "organization": "OpenAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gpt-4.1-2025-04-14", "score": 0.0355450236966824, "raw_score": 0.0355450236966824, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gpt-4.1-2025-04-14", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gpt-4.1-2025-04-14", "source_model_name": "GPT-4.1", "reported_confidence_interval": 0.0101990946312181, "source_date": "2025-06-13T00:00:00Z" }, { "id": "52f7e71ff659c78d146e", "model": "o4 mini", "organization": "", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version o4-mini-2025-04-16", "score": 0.0331753554502369, "raw_score": 0.0331753554502369, "raw_scale": 1, "source_record": "browsecomp|v1|Score|o4-mini-2025-04-16", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "o4-mini-2025-04-16", "source_model_name": "o4 mini", "reported_confidence_interval": 0.0098653588357216, "source_date": "2025-06-13T00:00:00Z" }, { "id": "a39c37a9621aa20a05a8", "model": "DeepSeek-R1", "organization": "DeepSeek", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version deepseek-r1-0528", "score": 0.0308056872037914, "raw_score": 0.0308056872037914, "raw_scale": 1, "source_record": "browsecomp|v1|Score|deepseek-r1-0528", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "deepseek-r1-0528", "source_model_name": "DeepSeek-R1", "reported_confidence_interval": 0.009518140634117, "source_date": "2025-07-02T00:00:00Z" }, { "id": "0bec07078789535ea199", "model": "deepseek-v3.1", "organization": "DeepSeek", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version deepseek-v3.1", "score": 0.026066350710900472, "raw_score": 0.026066350710900472, "raw_scale": 1, "source_record": "browsecomp|v1|Score|deepseek-v3.1", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "deepseek-v3.1", "source_model_name": "", "reported_confidence_interval": 0.008776796739938603, "source_date": "2025-08-27T00:00:00Z" }, { "id": "f49cdf54ee20c3370133", "model": "Gemini 2.5 Flash Preview", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemini-2.5-flash-preview-05-20", "score": 0.0252764612954186, "raw_score": 0.0252764612954186, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemini-2.5-flash-preview-05-20", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemini-2.5-flash-preview-05-20", "source_model_name": "Gemini 2.5 Flash Preview", "reported_confidence_interval": 0.0086462960454222, "source_date": "2025-06-13T00:00:00Z" }, { "id": "92b7061e32b9b084944f", "model": "Grok 3 Mini", "organization": "xAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version grok-3-mini", "score": 0.0173775671406003, "raw_score": 0.0173775671406003, "raw_scale": 1, "source_record": "browsecomp|v1|Score|grok-3-mini", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "grok-3-mini", "source_model_name": "Grok 3 Mini", "reported_confidence_interval": 0.0071981196780867, "source_date": "2025-06-13T00:00:00Z" }, { "id": "94f2a64432aa0454873e", "model": "gpt-oss-120b", "organization": "OpenAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gpt-oss-120b", "score": 0.0157977883096366, "raw_score": 0.0157977883096366, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gpt-oss-120b", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gpt-oss-120b", "source_model_name": "gpt-oss-120b", "reported_confidence_interval": 0.00686865260213, "source_date": "2025-08-21T00:00:00Z" }, { "id": "1b68dfcfd71cde13f0a4", "model": "o3 mini", "organization": "", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version o3-mini-2025-01-31", "score": 0.0150078988941548, "raw_score": 0.0150078988941548, "raw_scale": 1, "source_record": "browsecomp|v1|Score|o3-mini-2025-01-31", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "o3-mini-2025-01-31", "source_model_name": "o3 mini", "reported_confidence_interval": 0.0066974203856263, "source_date": "2025-06-13T00:00:00Z" }, { "id": "b36614709c18657d771f", "model": "DeepSeek-V3", "organization": "DeepSeek", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version deepseek-v3", "score": 0.0126382306477093, "raw_score": 0.0126382306477093, "raw_scale": 1, "source_record": "browsecomp|v1|Score|deepseek-v3", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "deepseek-v3", "source_model_name": "DeepSeek-V3", "reported_confidence_interval": 0.0061533629224853, "source_date": "2025-06-13T00:00:00Z" }, { "id": "f35600219063630ed565", "model": "Grok 3", "organization": "xAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version grok-3", "score": 0.0126382306477093, "raw_score": 0.0126382306477093, "raw_scale": 1, "source_record": "browsecomp|v1|Score|grok-3", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "grok-3", "source_model_name": "Grok 3", "reported_confidence_interval": 0.0061533629224853, "source_date": "2025-06-13T00:00:00Z" }, { "id": "d594af3581a42ac37d14", "model": "gpt-oss-20b", "organization": "OpenAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gpt-oss-20b", "score": 0.0126382306477093, "raw_score": 0.0126382306477093, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gpt-oss-20b", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gpt-oss-20b", "source_model_name": "", "reported_confidence_interval": 0.0061533629224853, "source_date": "2025-08-21T00:00:00Z" }, { "id": "6c674a5e20187f3b5c7b", "model": "Gemini 3.1 Flash-Lite Preview", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemini-3.1-flash-lite-preview", "score": 0.0110584518167456, "raw_score": 0.0110584518167456, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemini-3.1-flash-lite-preview", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemini-3.1-flash-lite-preview", "source_model_name": "Gemini 3.1 Flash-Lite Preview", "reported_confidence_interval": 0.0057605468728409 }, { "id": "5eee4840e5209922235a", "model": "Claude Opus 4", "organization": "Anthropic", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version claude-opus-4-20250514", "score": 0.0102685624012638, "raw_score": 0.0102685624012638, "raw_scale": 1, "source_record": "browsecomp|v1|Score|claude-opus-4-20250514", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "claude-opus-4-20250514", "source_model_name": "Claude Opus 4", "reported_confidence_interval": 0.0055532182743072, "source_date": "2025-06-13T00:00:00Z" }, { "id": "5cdf34f125648d4b1472", "model": "Claude Opus 4.1", "organization": "Anthropic", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version claude-opus-4-1-20250805", "score": 0.0102685624012638, "raw_score": 0.0102685624012638, "raw_scale": 1, "source_record": "browsecomp|v1|Score|claude-opus-4-1-20250805", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "claude-opus-4-1-20250805", "source_model_name": "Claude Opus 4.1", "reported_confidence_interval": 0.0055532182743072, "source_date": "2025-09-03T00:00:00Z" }, { "id": "e998c7c6ab7449f68da2", "model": "Grok 2", "organization": "xAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version grok-2-1212", "score": 0.0102685624012638, "raw_score": 0.0102685624012638, "raw_scale": 1, "source_record": "browsecomp|v1|Score|grok-2-1212", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "grok-2-1212", "source_model_name": "Grok 2", "reported_confidence_interval": 0.0055532182743072, "source_date": "2025-06-13T00:00:00Z" }, { "id": "754b4818b41bf93020e8", "model": "Claude 3.5 Sonnet", "organization": "Anthropic", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version claude-3-5-sonnet-20241022", "score": 0.0094786729857819, "raw_score": 0.0094786729857819, "raw_scale": 1, "source_record": "browsecomp|v1|Score|claude-3-5-sonnet-20241022", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "claude-3-5-sonnet-20241022", "source_model_name": "Claude 3.5 Sonnet", "reported_confidence_interval": 0.0053374881491649, "source_date": "2025-06-13T00:00:00Z" }, { "id": "49778b1e5e66ee0b224b", "model": "o1 mini", "organization": "", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version o1-mini-2024-09-12", "score": 0.0094786729857819, "raw_score": 0.0094786729857819, "raw_scale": 1, "source_record": "browsecomp|v1|Score|o1-mini-2024-09-12", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "o1-mini-2024-09-12", "source_model_name": "o1 mini", "reported_confidence_interval": 0.0053374881491649, "source_date": "2025-06-13T00:00:00Z" }, { "id": "12496dddb19fc5708161", "model": "Claude 3.7 Sonnet", "organization": "Anthropic", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version claude-3-7-sonnet-20250219", "score": 0.0086887835703001, "raw_score": 0.0086887835703001, "raw_scale": 1, "source_record": "browsecomp|v1|Score|claude-3-7-sonnet-20250219", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "claude-3-7-sonnet-20250219", "source_model_name": "Claude 3.7 Sonnet", "reported_confidence_interval": 0.005112293017358, "source_date": "2025-06-13T00:00:00Z" }, { "id": "ae10ff6564b7bdf0a64e", "model": "GPT-4o", "organization": "OpenAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gpt-4o-2024-08-06", "score": 0.0071090047393364, "raw_score": 0.0071090047393364, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gpt-4o-2024-08-06", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gpt-4o-2024-08-06", "source_model_name": "GPT-4o", "reported_confidence_interval": 0.004627926213592, "source_date": "2025-06-13T00:00:00Z" }, { "id": "e8ad127f5ae9b6ea98ca", "model": "Claude Sonnet 4", "organization": "Anthropic", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version claude-sonnet-4-20250514", "score": 0.0063191153238546, "raw_score": 0.0063191153238546, "raw_scale": 1, "source_record": "browsecomp|v1|Score|claude-sonnet-4-20250514", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "claude-sonnet-4-20250514", "source_model_name": "Claude Sonnet 4", "reported_confidence_interval": 0.0043649859139208, "source_date": "2025-06-13T00:00:00Z" }, { "id": "2af11f34b6d045062007", "model": "Codestral", "organization": "", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version codestral-2501", "score": 0.0063191153238546, "raw_score": 0.0063191153238546, "raw_scale": 1, "source_record": "browsecomp|v1|Score|codestral-2501", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "codestral-2501", "source_model_name": "Codestral", "reported_confidence_interval": 0.0043649859139208, "source_date": "2025-07-02T00:00:00Z" }, { "id": "ee8432b80bccd99613a5", "model": "GPT-3.5 Turbo", "organization": "OpenAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gpt-3.5-turbo-1106", "score": 0.0063191153238546, "raw_score": 0.0063191153238546, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gpt-3.5-turbo-1106", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gpt-3.5-turbo-1106", "source_model_name": "GPT-3.5 Turbo", "reported_confidence_interval": 0.0043649859139208, "source_date": "2025-06-13T00:00:00Z" }, { "id": "a62744970200ac5c3b53", "model": "GPT-4o mini", "organization": "OpenAI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gpt-4o-mini-2024-07-18", "score": 0.0063191153238546, "raw_score": 0.0063191153238546, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gpt-4o-mini-2024-07-18", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gpt-4o-mini-2024-07-18", "source_model_name": "GPT-4o mini", "reported_confidence_interval": 0.0043649859139208, "source_date": "2025-06-13T00:00:00Z" }, { "id": "791f5de6e46d12c79018", "model": "Gemini 2.0 Flash", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemini-2.0-flash-001", "score": 0.0055292259083728, "raw_score": 0.0055292259083728, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemini-2.0-flash-001", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemini-2.0-flash-001", "source_model_name": "Gemini 2.0 Flash", "reported_confidence_interval": 0.0040846929666103, "source_date": "2025-06-13T00:00:00Z" }, { "id": "1d1dce9f0fb1ab27e944", "model": "Gemma 3 27B", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemma-3-27b-it", "score": 0.0055292259083728, "raw_score": 0.0055292259083728, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemma-3-27b-it", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemma-3-27b-it", "source_model_name": "Gemma 3 27B", "reported_confidence_interval": 0.0040846929666103, "source_date": "2025-06-13T00:00:00Z" }, { "id": "47d00dd6270de40de87f", "model": "Mistral Large 2", "organization": "Mistral AI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version mistral-large-2411", "score": 0.0055292259083728, "raw_score": 0.0055292259083728, "raw_scale": 1, "source_record": "browsecomp|v1|Score|mistral-large-2411", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "mistral-large-2411", "source_model_name": "Mistral Large 2", "reported_confidence_interval": 0.0040846929666103, "source_date": "2025-07-02T00:00:00Z" }, { "id": "9e9752585e0dd3cab0f4", "model": "Gemma 3 12B", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemma-3-12b-it", "score": 0.0047393364928909, "raw_score": 0.0047393364928909, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemma-3-12b-it", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemma-3-12b-it", "source_model_name": "Gemma 3 12B", "reported_confidence_interval": 0.0037831924147072, "source_date": "2025-06-13T00:00:00Z" }, { "id": "3aa3b35538154c06e645", "model": "Mixtral 8x22B", "organization": "Mistral AI", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version open-mixtral-8x22b-2404", "score": 0.0047393364928909, "raw_score": 0.0047393364928909, "raw_scale": 1, "source_record": "browsecomp|v1|Score|open-mixtral-8x22b-2404", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "open-mixtral-8x22b-2404", "source_model_name": "Mixtral 8x22B", "reported_confidence_interval": 0.0037831924147072, "source_date": "2025-07-02T00:00:00Z" }, { "id": "6940fc893cc8eec566da", "model": "Qwen 3 235B A22B Thinking", "organization": "Alibaba", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version qwen3-235b-a22b-thinking-2507", "score": 0.0047393364928909, "raw_score": 0.0047393364928909, "raw_scale": 1, "source_record": "browsecomp|v1|Score|qwen3-235b-a22b-thinking-2507", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "qwen3-235b-a22b-thinking-2507", "source_model_name": "Qwen 3 235B A22B Thinking", "reported_confidence_interval": 0.0037831924147072, "source_date": "2025-08-21T00:00:00Z" }, { "id": "ba9f8b137a9b219d7b64", "model": "Gemini 1.5 Flash", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemini-1.5-flash-002", "score": 0.0039494470774091, "raw_score": 0.0039494470774091, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemini-1.5-flash-002", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemini-1.5-flash-002", "source_model_name": "Gemini 1.5 Flash", "reported_confidence_interval": 0.0034549365659004, "source_date": "2025-06-13T00:00:00Z" }, { "id": "19e9f4167eb405017804", "model": "Gemini 1.5 Pro", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemini-1.5-pro-002", "score": 0.0039494470774091, "raw_score": 0.0039494470774091, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemini-1.5-pro-002", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemini-1.5-pro-002", "source_model_name": "Gemini 1.5 Pro", "reported_confidence_interval": 0.0034549365659004, "source_date": "2025-06-13T00:00:00Z" }, { "id": "64473628fb21e4863c78", "model": "Gemma 3 4B", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemma-3-4b-it", "score": 0.0039494470774091, "raw_score": 0.0039494470774091, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemma-3-4b-it", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemma-3-4b-it", "source_model_name": "Gemma 3 4B", "reported_confidence_interval": 0.0034549365659004, "source_date": "2025-06-13T00:00:00Z" }, { "id": "d24decd0fab77fe8ca51", "model": "Granite 4.0 Small", "organization": "", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version granite-4.0-h-small", "score": 0.0039494470774091, "raw_score": 0.0039494470774091, "raw_scale": 1, "source_record": "browsecomp|v1|Score|granite-4.0-h-small", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "granite-4.0-h-small", "source_model_name": "Granite 4.0 Small", "reported_confidence_interval": 0.0034549365659004, "source_date": "2025-09-30T00:00:00Z" }, { "id": "1127b3dd61c4b9723b35", "model": "Claude 3.5 Haiku", "organization": "Anthropic", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version claude-3-5-haiku-20241022", "score": 0.0031595576619273, "raw_score": 0.0031595576619273, "raw_scale": 1, "source_record": "browsecomp|v1|Score|claude-3-5-haiku-20241022", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "claude-3-5-haiku-20241022", "source_model_name": "Claude 3.5 Haiku", "reported_confidence_interval": 0.0030914142580004, "source_date": "2025-06-13T00:00:00Z" }, { "id": "cd2723592c327181088f", "model": "Ministral 8B", "organization": "", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version ministral-8b-2410", "score": 0.0031595576619273, "raw_score": 0.0031595576619273, "raw_scale": 1, "source_record": "browsecomp|v1|Score|ministral-8b-2410", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "ministral-8b-2410", "source_model_name": "Ministral 8B", "reported_confidence_interval": 0.0030914142580004, "source_date": "2025-07-02T00:00:00Z" }, { "id": "29430f586c6e1d3c8b2e", "model": "Ministral 3B", "organization": "", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version ministral-3b-2410", "score": 0.0015797788309636, "raw_score": 0.0015797788309636, "raw_scale": 1, "source_record": "browsecomp|v1|Score|ministral-3b-2410", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "ministral-3b-2410", "source_model_name": "Ministral 3B", "reported_confidence_interval": 0.002187691439014, "source_date": "2025-07-02T00:00:00Z" }, { "id": "732e29a736c587d0f43c", "model": "Gemini 1.5 Flash 8B", "organization": "Google", "configuration": "Kaggle BrowseComp v1; Score task v1; 1 attempt; latest run; version gemini-1.5-flash-8b-001", "score": 0.0007898894154818, "raw_score": 0.0007898894154818, "raw_scale": 1, "source_record": "browsecomp|v1|Score|gemini-1.5-flash-8b-001", "report_url": "https://www.kaggle.com/benchmarks/openai/browsecomp/leaderboard", "source_model_version": "gemini-1.5-flash-8b-001", "source_model_name": "Gemini 1.5 Flash 8B", "reported_confidence_interval": 0.001547543249777, "source_date": "2025-06-13T00:00:00Z" } ] }, { "id": "rsi-bench", "name": "RSI Bench", "group": "Agent 与工具", "unit": "score", "metric": "Mean reward", "protocol": { "en": "The leaderboard currently scores one task. Agents choose biomedical images and captions to train TinyCLIP. The trained model earns up to 50 points for matching images and captions and 50 for classifying images from text descriptions. Rankings use the mean across valid runs.", "zh": "当前总榜计入一道任务。Agent 为 TinyCLIP 挑选生物医学图文训练数据。训练后的 TinyCLIP 要把图像和图注配对，还要根据文字描述识别图像类别，两项各占 50 分。排名取每个参评模型有效运行的平均分。" }, "source": { "name": "Scale AI", "benchmark_creator": "Scale AI", "url": "https://rsi-benchmark.com/tasks", "data_url": "https://rsi-benchmark.com/tasks", "sha256": "937e453591a1a4c1cfdce21e61cf47609b40c12554ffeaf7fac81572268de2c9", "retrieved_at": "2026-11-02T07:10:42.022144+00:00", "selector": "All leaderboard, mean by model, harness and reasoning effort. Equal weight per scored task. Valid runs only.", "kind": "official_leaderboard", "status": "current_snapshot", "artifacts": [ { "data_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation", "sha256": "2929f5a8aca56a757d2b2c6c60b7206c67beb8dd7ea0060292d5e75fe0356d4d", "retrieved_at": "2026-11-02T07:11:17.980214+00:00" }, { "data_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation/verification", "sha256": "1445750fb4cc91d5dbbec3bc3d095e4a93692d4ef15f458d93c64719c5d2a7d3", "retrieved_at": "2026-11-02T07:12:02.444309+00:00" } ] }, "results": [ { "id": "27ad65a349cebd0b7d70", "model": "gpt-6-astra", "organization": "OpenAI", "configuration": "codex; reasoning=max", "score": 38.86, "raw_score": 38.86, "raw_scale": 1, "source_record": "All|gpt-6-astra|codex|max|mean", "report_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation#results", "harness": "codex", "reasoning_effort": "max", "valid_runs": 3, "excluded_runs": 0, "best_run": 40.27 }, { "id": "4623bb1c1e32cc7e9f75", "model": "gpt-6.1-sol", "organization": "OpenAI", "configuration": "codex; reasoning=max", "score": 36.97, "raw_score": 36.97, "raw_scale": 1, "source_record": "All|gpt-6.1-sol|codex|max|mean", "report_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation#results", "harness": "codex", "reasoning_effort": "max", "valid_runs": 3, "excluded_runs": 0, "best_run": 38.52 }, { "id": "69ac7f91eb3087577747", "model": "claude-opus-5-5", "organization": "Anthropic", "configuration": "claude-code; reasoning=max", "score": 35.24, "raw_score": 35.24, "raw_scale": 1, "source_record": "All|claude-opus-5-5|claude-code|max|mean", "report_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation#results", "harness": "claude-code", "reasoning_effort": "max", "valid_runs": 3, "excluded_runs": 0, "best_run": 36.01 }, { "id": "26956fe545658a1ed601", "model": "claude-sonnet-5-5", "organization": "Anthropic", "configuration": "claude-code; reasoning=max", "score": 35.22, "raw_score": 35.22, "raw_scale": 1, "source_record": "All|claude-sonnet-5-5|claude-code|max|mean", "report_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation#results", "harness": "claude-code", "reasoning_effort": "max", "valid_runs": 3, "excluded_runs": 0, "best_run": 36.96 }, { "id": "ca5f70813cb5d3463f6a", "model": "qwen3.8-max", "organization": "Alibaba", "configuration": "terminus-2; reasoning=max (= xhigh)", "score": 33.36, "raw_score": 33.36, "raw_scale": 1, "source_record": "All|qwen3.8-max|terminus-2|max (= xhigh)|mean", "report_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation#results", "harness": "terminus-2", "reasoning_effort": "max (= xhigh)", "valid_runs": 3, "excluded_runs": 0, "best_run": 34.4 }, { "id": "bd8d739ac0e962c52a54", "model": "kimi-k3", "organization": "Moonshot AI", "configuration": "terminus-2; reasoning=max", "score": 32.75, "raw_score": 32.75, "raw_scale": 1, "source_record": "All|kimi-k3|terminus-2|max|mean", "report_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation#results", "harness": "terminus-2", "reasoning_effort": "max", "valid_runs": 2, "excluded_runs": 1, "best_run": 32.76 }, { "id": "eacb6efc8d6c5668ea49", "model": "glm-5.3", "organization": "Z AI", "configuration": "terminus-2; reasoning=max", "score": 32.28, "raw_score": 32.28, "raw_scale": 1, "source_record": "All|glm-5.3|terminus-2|max|mean", "report_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation#results", "harness": "terminus-2", "reasoning_effort": "max", "valid_runs": 3, "excluded_runs": 0, "best_run": 32.78 }, { "id": "0b005833caa138de8130", "model": "deepseek-v4.1-flash", "organization": "DeepSeek", "configuration": "terminus-2; reasoning=max", "score": 32.15, "raw_score": 32.15, "raw_scale": 1, "source_record": "All|deepseek-v4.1-flash|terminus-2|max|mean", "report_url": "https://rsi-benchmark.com/tasks/biomedical-tinyclip-data-curation#results", "harness": "terminus-2", "reasoning_effort": "max", "valid_runs": 3, "excluded_runs": 0, "best_run": 32.81 } ], "score_max": 100, "score_decimals": 2 } ] }