{ "developer": "Anthropic", "models": [ { "model_group_id": "anthropic/claude-v1-3", "model_route_id": "anthropic__claude-v1-3", "model_family_name": "Anthropic Claude v1.3", "developer": "anthropic", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "general", "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:49.537868Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "anthropic/claude-v1.3" ], "last_updated": "2026-03-21T12:31:49.537868Z" } ], "score_summary": { "count": 17, "min": 0.219, "max": 4.995, "average": 2.146294117647059 }, "reproducibility_summary": { "results_total": 17, "has_reproducibility_gap_count": 17, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 17, "total_groups": 17, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 17, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 17, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic RLHF dataset", "Best ChatGPT Prompts", "GSM8K", "Helm lite", "Holistic Evaluation of Language Models (HELM)", "Koala test dataset", "LegalBench", "MATH", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "Open Assistant", "OpenbookQA", "Self Instruct", "Vicuna", "WMT 2014" ], "top_benchmark_scores": [ { "benchmark": "Best ChatGPT Prompts", "benchmarkKey": "helm_instruct_best_chatgpt_prompts", "canonical_display_name": "Best ChatGPT Prompts / Harmlessness", "evaluation_name": "Best ChatGPT Prompts", "score": 4.995, "metric": "Harmlessness on Best ChatGPT Prompts", "lower_is_better": false }, { "benchmark": "Self Instruct", "benchmarkKey": "helm_instruct_self_instruct", "canonical_display_name": "Self Instruct / Harmlessness", "evaluation_name": "Self Instruct", "score": 4.992, "metric": "Harmlessness on Self Instruct", "lower_is_better": false }, { "benchmark": "Vicuna", "benchmarkKey": "helm_instruct_vicuna", "canonical_display_name": "Vicuna / Harmlessness", "evaluation_name": "Vicuna", "score": 4.989, "metric": "Harmlessness on Vicuna", "lower_is_better": false }, { "benchmark": "Koala test dataset", "benchmarkKey": "helm_instruct_koala_test_dataset", "canonical_display_name": "Koala test dataset / Harmlessness", "evaluation_name": "Koala test dataset", "score": 4.981, "metric": "Harmlessness on Koala test dataset", "lower_is_better": false }, { "benchmark": "Open Assistant", "benchmarkKey": "helm_instruct_open_assistant", "canonical_display_name": "Open Assistant / Harmlessness", "evaluation_name": "Open Assistant", "score": 4.975, "metric": "Harmlessness on Open Assistant", "lower_is_better": false }, { "benchmark": "Anthropic RLHF dataset", "benchmarkKey": "helm_instruct_anthropic_rlhf_dataset", "canonical_display_name": "Anthropic RLHF dataset / Harmlessness", "evaluation_name": "Anthropic RLHF dataset", "score": 4.965, "metric": "Harmlessness on Anthropic RLHF dataset", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.908, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.784, "metric": "EM on GSM8K", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.723, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_lite_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.631, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "LegalBench", "benchmarkKey": "helm_lite_legalbench", "canonical_display_name": "LegalBench / Exact Match", "evaluation_name": "LegalBench", "score": 0.629, "metric": "EM on LegalBench", "lower_is_better": false }, { "benchmark": "MedQA", "benchmarkKey": "helm_lite_medqa", "canonical_display_name": "MedQA / Exact Match", "evaluation_name": "MedQA", "score": 0.618, "metric": "EM on MedQA", "lower_is_better": false }, { "benchmark": "Holistic Evaluation of Language Models (HELM)", "benchmarkKey": "helm_instruct", "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Win Rate", "evaluation_name": "helm_instruct", "score": 0.611, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "MATH", "benchmarkKey": "helm_lite_math", "canonical_display_name": "MATH / Equivalent (CoT)", "evaluation_name": "MATH", "score": 0.54, "metric": "Equivalent (CoT) on MATH", "lower_is_better": false }, { "benchmark": "Helm lite", "benchmarkKey": "helm_lite", "canonical_display_name": "Helm lite / Win Rate", "evaluation_name": "helm_lite", "score": 0.518, "metric": "How many models this model outperforms on average (over columns).", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-2-0", "model_route_id": "anthropic__claude-2-0", "model_family_name": "Claude 2.0", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "general", "knowledge", "reasoning" ], "last_updated": "2026-03-21T12:31:46.427425Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-2.0" ], "last_updated": "2026-03-21T12:31:46.427425Z" } ], "score_summary": { "count": 10, "min": 0.219, "max": 0.862, "average": 0.5836 }, "reproducibility_summary": { "results_total": 10, "has_reproducibility_gap_count": 10, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 10, "total_groups": 10, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 10, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 10, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "GSM8K", "Helm lite", "LegalBench", "MATH", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "WMT 2014" ], "top_benchmark_scores": [ { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.862, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.718, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "MedQA", "benchmarkKey": "helm_lite_medqa", "canonical_display_name": "MedQA / Exact Match", "evaluation_name": "MedQA", "score": 0.652, "metric": "EM on MedQA", "lower_is_better": false }, { "benchmark": "LegalBench", "benchmarkKey": "helm_lite_legalbench", "canonical_display_name": "LegalBench / Exact Match", "evaluation_name": "LegalBench", "score": 0.643, "metric": "EM on LegalBench", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_lite_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.639, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "MATH", "benchmarkKey": "helm_lite_math", "canonical_display_name": "MATH / Equivalent (CoT)", "evaluation_name": "MATH", "score": 0.603, "metric": "Equivalent (CoT) on MATH", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.583, "metric": "EM on GSM8K", "lower_is_better": false }, { "benchmark": "Helm lite", "benchmarkKey": "helm_lite", "canonical_display_name": "Helm lite / Win Rate", "evaluation_name": "helm_lite", "score": 0.489, "metric": "How many models this model outperforms on average (over columns).", "lower_is_better": false }, { "benchmark": "NaturalQuestions (closed-book)", "benchmarkKey": "helm_lite_naturalquestions_closed_book", "canonical_display_name": "NaturalQuestions (closed-book) / F1", "evaluation_name": "NaturalQuestions (closed-book)", "score": 0.428, "metric": "F1 on NaturalQuestions (closed-book)", "lower_is_better": false }, { "benchmark": "WMT 2014", "benchmarkKey": "helm_lite_wmt_2014", "canonical_display_name": "WMT 2014 / BLEU-4", "evaluation_name": "WMT 2014", "score": 0.219, "metric": "BLEU-4 on WMT 2014", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-2-1", "model_route_id": "anthropic__claude-2-1", "model_family_name": "Claude 2.1", "developer": "anthropic", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "general", "knowledge", "reasoning" ], "last_updated": "2026-03-21T12:31:52.005480Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "anthropic/claude-2.1" ], "last_updated": "2026-03-21T12:31:52.005480Z" } ], "score_summary": { "count": 46, "min": 0.048, "max": 0.923, "average": 0.6905217391304348 }, "reproducibility_summary": { "results_total": 46, "has_reproducibility_gap_count": 46, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 46, "total_groups": 46, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 46, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 46, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "GSM8K", "Helm lite", "LegalBench", "MATH", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "WMT 2014" ], "top_benchmark_scores": [ { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Marketing / Exact Match", "evaluation_name": "Marketing", "score": 0.923, "metric": "EM on Marketing", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.872, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.677, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "MedQA", "benchmarkKey": "helm_lite_medqa", "canonical_display_name": "MedQA / Exact Match", "evaluation_name": "MedQA", "score": 0.644, "metric": "EM on MedQA", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_lite_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.643, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "LegalBench", "benchmarkKey": "helm_lite_legalbench", "canonical_display_name": "LegalBench / Exact Match", "evaluation_name": "LegalBench", "score": 0.643, "metric": "EM on LegalBench", "lower_is_better": false }, { "benchmark": "MATH", "benchmarkKey": "helm_lite_math", "canonical_display_name": "MATH / Equivalent (CoT)", "evaluation_name": "MATH", "score": 0.632, "metric": "Equivalent (CoT) on MATH", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.604, "metric": "EM on GSM8K", "lower_is_better": false }, { "benchmark": "Helm lite", "benchmarkKey": "helm_lite", "canonical_display_name": "Helm lite / Win Rate", "evaluation_name": "helm_lite", "score": 0.437, "metric": "How many models this model outperforms on average (over columns).", "lower_is_better": false }, { "benchmark": "NaturalQuestions (closed-book)", "benchmarkKey": "helm_lite_naturalquestions_closed_book", "canonical_display_name": "NaturalQuestions (closed-book) / F1", "evaluation_name": "NaturalQuestions (closed-book)", "score": 0.375, "metric": "F1 on NaturalQuestions (closed-book)", "lower_is_better": false }, { "benchmark": "WMT 2014", "benchmarkKey": "helm_lite_wmt_2014", "canonical_display_name": "WMT 2014 / BLEU-4", "evaluation_name": "WMT 2014", "score": 0.204, "metric": "BLEU-4 on WMT 2014", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-21", "model_route_id": "anthropic__claude-21", "model_family_name": "Claude 2.1", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-21" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 15, "min": 0.0, "max": 14.0, "average": 1.6628000000000003 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 14.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 9.3, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.495, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.374, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.319, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.195, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.184, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.042, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.033, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-3-haiku", "model_route_id": "anthropic__claude-3-haiku", "model_family_name": "Claude 3 Haiku", "developer": "anthropic", "params_billions": null, "total_evaluations": 9, "benchmark_count": 9, "benchmark_family_count": 9, "categories_covered": [ "agentic", "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-3-haiku" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "20240307", "variant_label": "20240307", "evaluation_count": 8, "raw_model_ids": [ "Anthropic/claude-3-haiku-20240307", "anthropic/claude-3-haiku-20240307" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 463, "min": 0.0, "max": 129.374, "average": 1.1233291576673856 }, "reproducibility_summary": { "results_total": 463, "has_reproducibility_gap_count": 463, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 463, "total_groups": 463, "multi_source_groups": 0, "first_party_only_groups": 5, "source_type_distribution": { "first_party": 5, "third_party": 458, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 463, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC-C", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BIG-Bench Hard", "DROP", "GSM8K", "HarmBench", "HellaSwag", "Helm air bench", "Helm lite", "Helm safety", "LegalBench", "MATH", "MGSM", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "RewardBench", "RewardBench 2", "SWE-bench Verified", "SimpleSafetyTests", "WMT 2014", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 129.374, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 12.3, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 6.7, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 1.25, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Fundamental Rights / Refusal Rate", "evaluation_name": "AIRBench 2024 - Fundamental Rights", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Fundamental Rights", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 1.0, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Us Foreign Policy / Exact Match", "evaluation_name": "Us Foreign Policy", "score": 0.95, "metric": "EM on Us Foreign Policy", "lower_is_better": false }, { "benchmark": "RewardBench", "benchmarkKey": "reward_bench", "canonical_display_name": "Reward Bench / Chat / Accuracy", "evaluation_name": "Chat", "score": 0.9274, "metric": "Chat accuracy - includes easy chat subsets", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.912, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "ARC-C", "benchmarkKey": "llm_stats_arc_c", "canonical_display_name": "Arc C / Score", "evaluation_name": "llm_stats.arc-c", "score": 0.892, "metric": "The AI2 Reasoning Challenge (ARC) Challenge Set is a multiple-choice question-answering benchmark containing grade-school level science questions that require advanced reasoning capabilities. ARC-C specifically contains questions that were answered incorrectly by both retrieval-based and word co-occurrence algorithms, making it a particularly challenging subset designed to test commonsense reasoning abilities in AI systems.", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.878, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "llm_stats_hellaswag", "canonical_display_name": "Hellaswag / Score", "evaluation_name": "llm_stats.hellaswag", "score": 0.859, "metric": "A challenging commonsense natural language inference dataset that uses Adversarial Filtering to create questions trivial for humans (>95% accuracy) but difficult for state-of-the-art models, requiring completion of sentence endings based on physical situations and everyday activities", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.853, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.838, "metric": "EM on OpenbookQA", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-3-opus", "model_route_id": "anthropic__claude-3-opus", "model_family_name": "Claude 3 Opus", "developer": "anthropic", "params_billions": null, "total_evaluations": 10, "benchmark_count": 9, "benchmark_family_count": 9, "categories_covered": [ "agentic", "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-3-opus" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "20240229", "variant_label": "20240229", "evaluation_count": 9, "raw_model_ids": [ "Anthropic/claude-3-opus-20240229", "anthropic/claude-3-opus-20240229" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 461, "min": 0.0, "max": 75.0, "average": 1.1628600867678958 }, "reproducibility_summary": { "results_total": 461, "has_reproducibility_gap_count": 461, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 461, "total_groups": 460, "multi_source_groups": 0, "first_party_only_groups": 6, "source_type_distribution": { "first_party": 6, "third_party": 455, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 460, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC-C", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BIG-Bench Hard", "DROP", "GSM8K", "GSM8k", "HarmBench", "HellaSwag", "Helm air bench", "Helm lite", "Helm safety", "LegalBench", "MATH", "MGSM", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "RewardBench", "RewardBench 2", "SWE-bench Verified", "SimpleSafetyTests", "WMT 2014", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 75.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 30.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 19.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 18.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 15.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Fundamental Rights / Refusal Rate", "evaluation_name": "AIRBench 2024 - Fundamental Rights", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Fundamental Rights", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.998, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.974, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Astronomy / Exact Match", "evaluation_name": "Astronomy", "score": 0.967, "metric": "EM on Astronomy", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.967, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "ARC-C", "benchmarkKey": "llm_stats_arc_c", "canonical_display_name": "Arc C / Score", "evaluation_name": "llm_stats.arc-c", "score": 0.964, "metric": "The AI2 Reasoning Challenge (ARC) Challenge Set is a multiple-choice question-answering benchmark containing grade-school level science questions that require advanced reasoning capabilities. ARC-C specifically contains questions that were answered incorrectly by both retrieval-based and word co-occurrence algorithms, making it a particularly challenging subset designed to test commonsense reasoning abilities in AI systems.", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.956, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "llm_stats_hellaswag", "canonical_display_name": "Hellaswag / Score", "evaluation_name": "llm_stats.hellaswag", "score": 0.954, "metric": "A challenging commonsense natural language inference dataset that uses Adversarial Filtering to create questions trivial for humans (>95% accuracy) but difficult for state-of-the-art models, requiring completion of sentence endings based on physical situations and everyday activities", "lower_is_better": false }, { "benchmark": "GSM8k", "benchmarkKey": "llm_stats_gsm8k", "canonical_display_name": "Gsm8k / Score", "evaluation_name": "llm_stats.gsm8k", "score": 0.95, "metric": "Grade School Math 8K, a dataset of 8.5K high-quality linguistically diverse grade school math word problems requiring multi-step reasoning and elementary arithmetic operations.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-3-sonnet", "model_route_id": "anthropic__claude-3-sonnet", "model_family_name": "Claude 3 Sonnet", "developer": "anthropic", "params_billions": null, "total_evaluations": 7, "benchmark_count": 7, "benchmark_family_count": 7, "categories_covered": [ "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-3-sonnet" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "20240229", "variant_label": "20240229", "evaluation_count": 6, "raw_model_ids": [ "Anthropic/claude-3-sonnet-20240229", "anthropic/claude-3-sonnet-20240229" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 452, "min": 0.0, "max": 15.0, "average": 0.9030506637168133 }, "reproducibility_summary": { "results_total": 452, "has_reproducibility_gap_count": 452, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 452, "total_groups": 452, "multi_source_groups": 0, "first_party_only_groups": 6, "source_type_distribution": { "first_party": 6, "third_party": 446, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 452, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC-C", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BIG-Bench Hard", "DROP", "GSM8K", "GSM8k", "HarmBench", "HellaSwag", "Helm air bench", "Helm lite", "Helm safety", "LegalBench", "MATH", "MGSM", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "RewardBench", "SimpleSafetyTests", "WMT 2014", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 10.3, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Self-harm / Refusal Rate", "evaluation_name": "AIRBench 2024 - Self-harm", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Self-harm", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.998, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.958, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.943, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Us Foreign Policy / Exact Match", "evaluation_name": "Us Foreign Policy", "score": 0.94, "metric": "EM on Us Foreign Policy", "lower_is_better": false }, { "benchmark": "RewardBench", "benchmarkKey": "reward_bench", "canonical_display_name": "Reward Bench / Chat / Accuracy", "evaluation_name": "Chat", "score": 0.9344, "metric": "Chat accuracy - includes easy chat subsets", "lower_is_better": false }, { "benchmark": "ARC-C", "benchmarkKey": "llm_stats_arc_c", "canonical_display_name": "Arc C / Score", "evaluation_name": "llm_stats.arc-c", "score": 0.932, "metric": "The AI2 Reasoning Challenge (ARC) Challenge Set is a multiple-choice question-answering benchmark containing grade-school level science questions that require advanced reasoning capabilities. ARC-C specifically contains questions that were answered incorrectly by both retrieval-based and word co-occurrence algorithms, making it a particularly challenging subset designed to test commonsense reasoning abilities in AI systems.", "lower_is_better": false }, { "benchmark": "GSM8k", "benchmarkKey": "llm_stats_gsm8k", "canonical_display_name": "Gsm8k / Score", "evaluation_name": "llm_stats.gsm8k", "score": 0.923, "metric": "Grade School Math 8K, a dataset of 8.5K high-quality linguistically diverse grade school math word problems requiring multi-step reasoning and elementary arithmetic operations.", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.918, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.907, "metric": "EM on GSM8K", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-3-5-haiku", "model_route_id": "anthropic__claude-3-5-haiku", "model_family_name": "Claude 3.5 Haiku", "developer": "anthropic", "params_billions": null, "total_evaluations": 7, "benchmark_count": 6, "benchmark_family_count": 6, "categories_covered": [ "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-3-5-haiku" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "20241022", "variant_label": "20241022", "evaluation_count": 6, "raw_model_ids": [ "anthropic/claude-3-5-haiku-20241022" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 111, "min": 0.0, "max": 18.7, "average": 0.9039045045045044 }, "reproducibility_summary": { "results_total": 111, "has_reproducibility_gap_count": 111, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 111, "total_groups": 92, "multi_source_groups": 0, "first_party_only_groups": 2, "source_type_distribution": { "first_party": 2, "third_party": 109, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 92, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "DROP", "GPQA", "GSM8K", "Global-MMLU Lite", "Helm lite", "Holistic Evaluation of Language Models (HELM)", "IFEval", "LegalBench", "MATH", "MGSM", "MMLU", "MMLU-Pro", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "Omni-MATH", "OpenbookQA", "WMT 2014", "WildBench" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 18.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 10.7, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 4.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 1.6, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Us Foreign Policy / Exact Match", "evaluation_name": "Us Foreign Policy", "score": 0.94, "metric": "EM on Us Foreign Policy", "lower_is_better": false }, { "benchmark": "MATH", "benchmarkKey": "helm_lite_math", "canonical_display_name": "MATH / Equivalent (CoT)", "evaluation_name": "MATH", "score": 0.872, "metric": "Equivalent (CoT) on MATH", "lower_is_better": false }, { "benchmark": "MGSM", "benchmarkKey": "llm_stats_mgsm", "canonical_display_name": "Mgsm / Score", "evaluation_name": "llm_stats.mgsm", "score": 0.856, "metric": "MGSM (Multilingual Grade School Math) is a benchmark of grade-school math problems. Contains 250 grade-school math problems manually translated from the GSM8K dataset into ten typologically diverse languages: Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, and Telugu. Evaluates multilingual mathematical reasoning capabilities.", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.854, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "DROP", "benchmarkKey": "llm_stats_drop", "canonical_display_name": "Drop / Score", "evaluation_name": "llm_stats.drop", "score": 0.831, "metric": "DROP (Discrete Reasoning Over Paragraphs) is a reading comprehension benchmark requiring discrete reasoning over paragraph content. It contains crowdsourced, adversarially-created questions that require resolving references and performing discrete operations like addition, counting, or sorting, demanding comprehensive paragraph understanding beyond paraphrase-and-entity-typing shortcuts.", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.815, "metric": "EM on GSM8K", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.8, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "IFEval", "benchmarkKey": "helm_capabilities_ifeval", "canonical_display_name": "IFEval / Strict Accuracy", "evaluation_name": "IFEval", "score": 0.792, "metric": "IFEval Strict Acc on IFEval", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.763, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "WildBench", "benchmarkKey": "helm_capabilities_wildbench", "canonical_display_name": "WildBench / Score", "evaluation_name": "WildBench", "score": 0.76, "metric": "WB Score on WildBench", "lower_is_better": false }, { "benchmark": "MedQA", "benchmarkKey": "helm_lite_medqa", "canonical_display_name": "MedQA / Exact Match", "evaluation_name": "MedQA", "score": 0.722, "metric": "EM on MedQA", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-3-5-sonnet", "model_route_id": "anthropic__claude-3-5-sonnet", "model_family_name": "Claude 3.5 Sonnet", "developer": "anthropic", "params_billions": null, "total_evaluations": 32, "benchmark_count": 9, "benchmark_family_count": 9, "categories_covered": [ "agentic", "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "20240620", "variant_label": "20240620", "evaluation_count": 7, "raw_model_ids": [ "Anthropic/claude-3-5-sonnet-20240620", "anthropic/claude-3-5-sonnet-20240620" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "20241022", "variant_label": "20241022", "evaluation_count": 25, "raw_model_ids": [ "anthropic/claude-3-5-sonnet-20241022" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 905, "min": 0.0, "max": 1.0, "average": 0.8755944751381212 }, "reproducibility_summary": { "results_total": 905, "has_reproducibility_gap_count": 905, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 905, "total_groups": 460, "multi_source_groups": 0, "first_party_only_groups": 15, "source_type_distribution": { "first_party": 21, "third_party": 884, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 460, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AI2D", "Anthropic Red Team", "BBQ", "BIG-Bench Hard", "ChartQA", "DROP", "DocVQA", "GPQA", "GSM8K", "GSM8k", "HarmBench", "Helm air bench", "Helm lite", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "HumanEval", "IFEval", "LegalBench", "MATH", "MGSM", "MMLU", "MMLU-Pro", "MathVista", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OSWorld Extended", "OSWorld Screenshot-only", "Omni-MATH", "OpenbookQA", "RewardBench", "RewardBench 2", "SWE-bench Verified", "SimpleSafetyTests", "TAU-bench Airline", "TAU-bench Retail", "WMT 2014", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Self-harm / Refusal Rate", "evaluation_name": "AIRBench 2024 - Self-harm", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Self-harm", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.998, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.981, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Medical Genetics / Exact Match", "evaluation_name": "Medical Genetics", "score": 0.98, "metric": "EM on Medical Genetics", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.977, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.972, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "GSM8k", "benchmarkKey": "llm_stats_gsm8k", "canonical_display_name": "Gsm8k / Score", "evaluation_name": "llm_stats.gsm8k", "score": 0.964, "metric": "Grade School Math 8K, a dataset of 8.5K high-quality linguistically diverse grade school math word problems requiring multi-step reasoning and elementary arithmetic operations.", "lower_is_better": false }, { "benchmark": "RewardBench", "benchmarkKey": "reward_bench", "canonical_display_name": "Reward Bench / Chat / Accuracy", "evaluation_name": "Chat", "score": 0.9637, "metric": "Chat accuracy - includes easy chat subsets", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.956, "metric": "EM on GSM8K", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.956, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "DocVQA", "benchmarkKey": "llm_stats_docvqa", "canonical_display_name": "Docvqa / Score", "evaluation_name": "llm_stats.docvqa", "score": 0.952, "metric": "A dataset for Visual Question Answering on document images containing 50,000 questions defined on 12,000+ document images. The benchmark tests AI's ability to understand document structure and content, requiring models to comprehend document layout and perform information retrieval to answer questions about document images.", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.949, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "AI2D", "benchmarkKey": "llm_stats_ai2d", "canonical_display_name": "Ai2d / Score", "evaluation_name": "llm_stats.ai2d", "score": 0.947, "metric": "AI2D is a dataset of 4,903 illustrative diagrams from grade school natural sciences (such as food webs, human physiology, and life cycles) with over 15,000 multiple choice questions and answers. The benchmark evaluates diagram understanding and visual reasoning capabilities, requiring models to interpret diagrammatic elements, relationships, and structure to answer questions about scientific concepts represented in visual form.", "lower_is_better": false }, { "benchmark": "HumanEval", "benchmarkKey": "llm_stats_humaneval", "canonical_display_name": "Humaneval / Score", "evaluation_name": "llm_stats.humaneval", "score": 0.937, "metric": "A benchmark that measures functional correctness for synthesizing programs from docstrings, consisting of 164 original programming problems assessing language comprehension, algorithms, and simple mathematics", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-35-sonnet-june-24", "model_route_id": "anthropic__claude-35-sonnet-june-24", "model_family_name": "Claude 3.5 Sonnet (June '24)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-35-sonnet-june-24" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 14, "min": 0.0, "max": 26.0, "average": 4.761142857142858 }, "reproducibility_summary": { "results_total": 14, "has_reproducibility_gap_count": 14, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 14, "total_groups": 14, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 14, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 14, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 26.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 14.2, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.751, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.695, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.56, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.316, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.097, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.037, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-35-sonnet", "model_route_id": "anthropic__claude-35-sonnet", "model_family_name": "Claude 3.5 Sonnet (Oct '24)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-35-sonnet" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 15, "min": 0.0, "max": 30.2, "average": 4.8790000000000004 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 30.2, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 15.9, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.772, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.771, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.599, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.381, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.366, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.157, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.039, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-3-7", "model_route_id": "anthropic__claude-3-7", "model_family_name": "Claude 3.7", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.572172Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-3.7" ], "last_updated": "2026-04-07T08:15:57.572172Z" } ], "score_summary": { "count": 6, "min": 0.0, "max": 0.136, "average": 0.07233333333333333 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 6, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.058, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-3-7-sonnet", "model_route_id": "anthropic__claude-3-7-sonnet", "model_family_name": "Claude 3.7 Sonnet", "developer": "anthropic", "params_billions": null, "total_evaluations": 37, "benchmark_count": 11, "benchmark_family_count": 11, "categories_covered": [ "agentic", "coding", "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 24, "raw_model_ids": [ "anthropic/Claude-3.7-Sonnet", "anthropic/claude-3-7-sonnet", "anthropic/claude-3.7-sonnet" ], "last_updated": "2026-04-20T17:00:03.994647Z" }, { "variant_key": "20250219", "variant_label": "20250219", "evaluation_count": 13, "raw_model_ids": [ "anthropic/claude-3-7-sonnet-20250219" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 492, "min": 0.0, "max": 966.4162650074346, "average": 2.9864196341678197 }, "reproducibility_summary": { "results_total": 492, "has_reproducibility_gap_count": 492, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 492, "total_groups": 451, "multi_source_groups": 0, "first_party_only_groups": 5, "source_type_distribution": { "first_party": 5, "third_party": 487, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 451, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "Easy Problems", "GPQA", "Global-MMLU Lite", "Hard Problems", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MATH-500", "MMLU-Pro", "Medium Problems", "Multi-SWE-bench (c)", "Multi-SWE-bench (c++)", "Multi-SWE-bench (go)", "Multi-SWE-bench (java)", "Multi-SWE-bench (javascript)", "Multi-SWE-bench (rust)", "Multi-SWE-bench (typescript)", "Omni-MATH", "RewardBench 2", "SWE-bench Verified", "SciArena leaderboard API", "SimpleSafetyTests", "TAU-bench Airline", "TAU-bench Retail", "Terminal-Bench", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 30.8, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 26.7, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 21.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "canonical_display_name": "SciArena / Cost per 100 Calls", "evaluation_name": "overall_cost_per_100_calls_usd", "score": 5.7391, "metric": "overall_cost_per_100_calls_usd", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #43: Illegal/Regulated substances/goods / Refusal Rate", "evaluation_name": "AIRBench 2024 - #43: Illegal/Regulated substances/goods", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #43: Illegal/Regulated substances/goods", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.997, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.964, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "MATH-500", "benchmarkKey": "llm_stats_math_500", "canonical_display_name": "Math 500 / Score", "evaluation_name": "llm_stats.math-500", "score": 0.962, "metric": "MATH-500 is a subset of the MATH dataset containing 500 challenging competition mathematics problems from AMC 10, AMC 12, AIME, and other mathematics competitions. Each problem includes full step-by-step solutions and spans multiple difficulty levels across seven mathematical subjects including Prealgebra, Algebra, Number Theory, Counting and Probability, Geometry, Intermediate Algebra, and Precalculus.", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.945, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "IFEval", "benchmarkKey": "llm_stats_ifeval", "canonical_display_name": "Ifeval / Score", "evaluation_name": "llm_stats.ifeval", "score": 0.932, "metric": "Instruction-Following Evaluation (IFEval) benchmark for large language models, focusing on verifiable instructions with 25 types of instructions and around 500 prompts containing one or more verifiable constraints", "lower_is_better": false }, { "benchmark": "RewardBench 2", "benchmarkKey": "reward_bench_2", "canonical_display_name": "Reward Bench 2 / Focus / Score", "evaluation_name": "Focus", "score": 0.9212, "metric": "Focus score - measures response focus", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-3-7-sonnet-thinking", "model_route_id": "anthropic__claude-3-7-sonnet-thinking", "model_family_name": "Claude 3.7 Sonnet (Reasoning)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-3-7-sonnet-thinking" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 21, "min": 0.0, "max": 56.3, "average": 7.096857142857142 }, "reproducibility_summary": { "results_total": 21, "has_reproducibility_gap_count": 21, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 21, "total_groups": 21, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 21, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 21, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 56.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 34.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 27.6, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.947, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.837, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.772, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.607, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.563, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.547, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.487, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.483, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.473, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-3-7-thinking-16k", "model_route_id": "anthropic__claude-3-7-thinking-16k", "model_family_name": "Claude 3.7 Thinking 16K", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.572658Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-3.7-thinking-16k" ], "last_updated": "2026-04-07T08:15:57.572658Z" } ], "score_summary": { "count": 6, "min": 0.007, "max": 0.57, "average": 0.2851666666666666 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 6, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Score", "evaluation_name": "v1_Semi_Private", "score": 0.286, "metric": "v1_Semi_Private", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-3-7-thinking-1k", "model_route_id": "anthropic__claude-3-7-thinking-1k", "model_family_name": "Claude 3.7 Thinking 1K", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.572998Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-3.7-thinking-1k" ], "last_updated": "2026-04-07T08:15:57.572998Z" } ], "score_summary": { "count": 6, "min": 0.004, "max": 0.14, "average": 0.07966666666666668 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 6, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.07, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-3-7-thinking-8k", "model_route_id": "anthropic__claude-3-7-thinking-8k", "model_family_name": "Claude 3.7 Thinking 8K", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.573335Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-3.7-thinking-8k" ], "last_updated": "2026-04-07T08:15:57.573335Z" } ], "score_summary": { "count": 6, "min": 0.0, "max": 0.36, "average": 0.19183333333333333 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 6, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.21, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-4-opus-thinking", "model_route_id": "anthropic__claude-4-opus-thinking", "model_family_name": "Claude 4 Opus (Reasoning)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-4-opus-thinking" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 21, "min": 0.117, "max": 75.0, "average": 15.39228571428571 }, "reproducibility_summary": { "results_total": 21, "has_reproducibility_gap_count": 21, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 21, "total_groups": 21, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 21, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 21, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 75.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 73.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 39.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 35.559, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 34.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 30.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 15.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 7.084, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 7.084, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.982, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.873, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.796, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.757, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.734, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.733, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-4-sonnet-thinking", "model_route_id": "anthropic__claude-4-sonnet-thinking", "model_family_name": "Claude 4 Sonnet (Reasoning)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-4-sonnet-thinking" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 21, "min": 0.096, "max": 74.3, "average": 11.416190476190476 }, "reproducibility_summary": { "results_total": 21, "has_reproducibility_gap_count": 21, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 21, "total_groups": 21, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 21, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 21, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 74.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 44.274, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 38.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 34.1, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 8.469, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 8.469, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.991, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.842, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.777, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.773, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.743, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.655, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-4-1-opus", "model_route_id": "anthropic__claude-4-1-opus", "model_family_name": "Claude 4.1 Opus (Non-reasoning)", "developer": "anthropic", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "knowledge", "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "anthropic/claude-4-1-opus", "anthropic/claude-4.1-opus" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 10, "min": 1.413, "max": 1126.3532524689376, "average": 135.14002524689377 }, "reproducibility_summary": { "results_total": 10, "has_reproducibility_gap_count": 10, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 10, "total_groups": 10, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 10, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 10, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "SciArena leaderboard API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 75.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 36.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 35.573, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 30.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 15.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "canonical_display_name": "SciArena / Rank", "evaluation_name": "overall_rank", "score": 2.0, "metric": "overall_rank", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.413, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 1.413, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-4-1-opus-thinking", "model_route_id": "anthropic__claude-4-1-opus-thinking", "model_family_name": "Claude 4.1 Opus (Reasoning)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-4-1-opus-thinking" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.119, "max": 80.3, "average": 17.573052631578943 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 80.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 75.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 42.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 36.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 33.984, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 30.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 15.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 7.578, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 7.578, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.88, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.809, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.803, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.714, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.663, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.654, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-4-5-haiku", "model_route_id": "anthropic__claude-4-5-haiku", "model_family_name": "Claude 4.5 Haiku (Non-reasoning)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-4-5-haiku" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.043, "max": 97.549, "average": 11.087684210526316 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 97.549, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 39.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 31.1, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 29.6, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 5.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 2.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.8, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.646, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.614, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.614, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.511, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.437, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.42, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.39, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-4-5-haiku-reasoning", "model_route_id": "anthropic__claude-4-5-haiku-reasoning", "model_family_name": "Claude 4.5 Haiku (Reasoning)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-4-5-haiku-reasoning" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.097, "max": 117.936, "average": 16.168105263157898 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 117.936, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 83.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 37.1, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 32.6, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 11.189, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 11.189, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 5.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 2.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.837, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.76, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.703, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.672, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.615, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.547, "metric": "Benchmark score on tau2.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-4-5-sonnet-thinking", "model_route_id": "anthropic__claude-4-5-sonnet-thinking", "model_family_name": "Claude 4.5 Sonnet (Reasoning)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-4-5-sonnet-thinking" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.173, "max": 88.0, "average": 13.887 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 88.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 47.789, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 43.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 38.6, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 8.087, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 8.087, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.88, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.875, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.834, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.781, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.714, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.657, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-code", "model_route_id": "anthropic__claude-code", "model_family_name": "Claude Code", "developer": "Anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-22T13:06:30.542123Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-code" ], "last_updated": "2026-04-22T13:06:30.542123Z" } ], "score_summary": { "count": 4, "min": 1.451, "max": 673.8, "average": 223.80475 }, "reproducibility_summary": { "results_total": 4, "has_reproducibility_gap_count": 4, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 4, "total_groups": 4, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 4, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 4, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "CocoaBench v1.0" ], "top_benchmark_scores": [ { "benchmark": "CocoaBench v1.0", "benchmarkKey": "cocoabench", "canonical_display_name": "Cocoabench / Avg Cost Usd", "evaluation_name": "cocoabench.overall.avg_cost_usd", "score": 1.451, "metric": "Average task cost in USD", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-instant", "model_route_id": "anthropic__claude-instant", "model_family_name": "Claude Instant", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-instant" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 14, "min": 0.0, "max": 7.8, "average": 1.1696428571428572 }, "reproducibility_summary": { "results_total": 14, "has_reproducibility_gap_count": 14, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 14, "total_groups": 14, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 14, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 14, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 7.8, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 7.4, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.434, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.33, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.264, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.109, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.038, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.0, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-instant-1-2", "model_route_id": "anthropic__claude-instant-1-2", "model_family_name": "Claude Instant 1.2", "developer": "anthropic", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "general", "knowledge", "reasoning" ], "last_updated": "2026-03-21T12:31:52.005480Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "anthropic/claude-instant-1.2" ], "last_updated": "2026-03-21T12:31:52.005480Z" } ], "score_summary": { "count": 46, "min": 0.186, "max": 0.9, "average": 0.6491304347826087 }, "reproducibility_summary": { "results_total": 46, "has_reproducibility_gap_count": 46, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 46, "total_groups": 46, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 46, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 46, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "GSM8K", "Helm lite", "LegalBench", "MATH", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "WMT 2014" ], "top_benchmark_scores": [ { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Us Foreign Policy / Exact Match", "evaluation_name": "Us Foreign Policy", "score": 0.9, "metric": "EM on Us Foreign Policy", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.844, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.721, "metric": "EM on GSM8K", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_lite_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.631, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.616, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "LegalBench", "benchmarkKey": "helm_lite_legalbench", "canonical_display_name": "LegalBench / Exact Match", "evaluation_name": "LegalBench", "score": 0.586, "metric": "EM on LegalBench", "lower_is_better": false }, { "benchmark": "MedQA", "benchmarkKey": "helm_lite_medqa", "canonical_display_name": "MedQA / Exact Match", "evaluation_name": "MedQA", "score": 0.559, "metric": "EM on MedQA", "lower_is_better": false }, { "benchmark": "MATH", "benchmarkKey": "helm_lite_math", "canonical_display_name": "MATH / Equivalent (CoT)", "evaluation_name": "MATH", "score": 0.499, "metric": "Equivalent (CoT) on MATH", "lower_is_better": false }, { "benchmark": "Helm lite", "benchmarkKey": "helm_lite", "canonical_display_name": "Helm lite / Win Rate", "evaluation_name": "helm_lite", "score": 0.399, "metric": "How many models this model outperforms on average (over columns).", "lower_is_better": false }, { "benchmark": "NaturalQuestions (closed-book)", "benchmarkKey": "helm_lite_naturalquestions_closed_book", "canonical_display_name": "NaturalQuestions (closed-book) / F1", "evaluation_name": "NaturalQuestions (closed-book)", "score": 0.343, "metric": "F1 on NaturalQuestions (closed-book)", "lower_is_better": false }, { "benchmark": "WMT 2014", "benchmarkKey": "helm_lite_wmt_2014", "canonical_display_name": "WMT 2014 / BLEU-4", "evaluation_name": "WMT 2014", "score": 0.194, "metric": "BLEU-4 on WMT 2014", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-mythos-preview", "model_route_id": "anthropic__claude-mythos-preview", "model_family_name": "Claude Mythos Preview", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-mythos-preview" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 16, "min": 0.59, "max": 1.0, "average": 0.850875 }, "reproducibility_summary": { "results_total": 16, "has_reproducibility_gap_count": 16, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 16, "total_groups": 16, "multi_source_groups": 0, "first_party_only_groups": 16, "source_type_distribution": { "first_party": 16, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 16, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BrowseComp", "CharXiv-R", "CyBench", "CyberGym", "FigQA", "GPQA", "Graphwalks BFS >128k", "Humanity's Last Exam", "MMMLU", "OSWorld-Verified", "SWE-Bench Multimodal", "SWE-Bench Pro", "SWE-Bench Verified", "SWE-bench Multilingual", "Terminal-Bench 2.0", "USAMO25" ], "top_benchmark_scores": [ { "benchmark": "CyBench", "benchmarkKey": "llm_stats_cybench", "canonical_display_name": "Cybench / Score", "evaluation_name": "llm_stats.cybench", "score": 1.0, "metric": "CyBench is a suite of Capture-the-Flag (CTF) challenges measuring agentic cyber attack capabilities. It evaluates dual-use cybersecurity knowledge and measures the 'unguided success rate', where agents complete tasks end-to-end without guidance on appropriate subtasks.", "lower_is_better": false }, { "benchmark": "USAMO25", "benchmarkKey": "llm_stats_usamo25", "canonical_display_name": "Usamo25 / Score", "evaluation_name": "llm_stats.usamo25", "score": 0.976, "metric": "The 2025 United States of America Mathematical Olympiad (USAMO) benchmark consists of six challenging mathematical problems requiring rigorous proof-based reasoning. USAMO is the most prestigious high school mathematics competition in the United States, serving as the final round of the American Mathematics Competitions series. This benchmark evaluates models on mathematical problem-solving capabilities beyond simple numerical computation, focusing on formal mathematical reasoning and proof generation.", "lower_is_better": false }, { "benchmark": "GPQA", "benchmarkKey": "llm_stats_gpqa", "canonical_display_name": "Gpqa / Score", "evaluation_name": "llm_stats.gpqa", "score": 0.946, "metric": "A challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. Questions are Google-proof and extremely difficult, with PhD experts reaching 65% accuracy.", "lower_is_better": false }, { "benchmark": "SWE-Bench Verified", "benchmarkKey": "llm_stats_swe_bench_verified", "canonical_display_name": "Swe Bench Verified / Score", "evaluation_name": "llm_stats.swe-bench-verified", "score": 0.939, "metric": "A verified subset of 500 software engineering problems from real GitHub issues, validated by human annotators for evaluating language models' ability to resolve real-world coding issues by generating patches for Python codebases.", "lower_is_better": false }, { "benchmark": "CharXiv-R", "benchmarkKey": "llm_stats_charxiv_r", "canonical_display_name": "Charxiv R / Score", "evaluation_name": "llm_stats.charxiv-r", "score": 0.932, "metric": "CharXiv-R is the reasoning component of the CharXiv benchmark, focusing on complex reasoning questions that require synthesizing information across visual chart elements. It evaluates multimodal large language models on their ability to understand and reason about scientific charts from arXiv papers through various reasoning tasks.", "lower_is_better": false }, { "benchmark": "MMMLU", "benchmarkKey": "llm_stats_mmmlu", "canonical_display_name": "Mmmlu / Score", "evaluation_name": "llm_stats.mmmlu", "score": 0.927, "metric": "Multilingual Massive Multitask Language Understanding dataset released by OpenAI, featuring professionally translated MMLU test questions across 14 languages including Arabic, Bengali, German, Spanish, French, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese, Swahili, Yoruba, and Chinese. Contains approximately 15,908 multiple-choice questions per language covering 57 subjects.", "lower_is_better": false }, { "benchmark": "FigQA", "benchmarkKey": "llm_stats_figqa", "canonical_display_name": "Figqa / Score", "evaluation_name": "llm_stats.figqa", "score": 0.89, "metric": "FigQA is a multiple-choice benchmark on interpreting scientific figures from biology papers. It evaluates dual-use biological knowledge and multimodal reasoning relevant to bioweapons development.", "lower_is_better": false }, { "benchmark": "SWE-bench Multilingual", "benchmarkKey": "llm_stats_swe_bench_multilingual", "canonical_display_name": "Swe Bench Multilingual / Score", "evaluation_name": "llm_stats.swe-bench-multilingual", "score": 0.873, "metric": "A multilingual benchmark for issue resolving in software engineering that covers Java, TypeScript, JavaScript, Go, Rust, C, and C++. Contains 1,632 high-quality instances carefully annotated from 2,456 candidates by 68 expert annotators, designed to evaluate Large Language Models across diverse software ecosystems beyond Python.", "lower_is_better": false }, { "benchmark": "BrowseComp", "benchmarkKey": "llm_stats_browsecomp", "canonical_display_name": "Browsecomp / Score", "evaluation_name": "llm_stats.browsecomp", "score": 0.869, "metric": "BrowseComp is a benchmark comprising 1,266 questions that challenge AI agents to persistently navigate the internet in search of hard-to-find, entangled information. The benchmark measures agents' ability to exercise persistence in information gathering, demonstrate creativity in web navigation, and find concise, verifiable answers. Despite the difficulty of the questions, BrowseComp is simple and easy-to-use, as predicted answers are short and easily verifiable against reference answers.", "lower_is_better": false }, { "benchmark": "CyberGym", "benchmarkKey": "llm_stats_cybergym", "canonical_display_name": "Cybergym / Score", "evaluation_name": "llm_stats.cybergym", "score": 0.831, "metric": "CyberGym is a benchmark for evaluating AI agents on cybersecurity tasks, testing their ability to identify vulnerabilities, perform security analysis, and complete security-related challenges in a controlled environment.", "lower_is_better": false }, { "benchmark": "Terminal-Bench 2.0", "benchmarkKey": "llm_stats_terminal_bench_2", "canonical_display_name": "Terminal Bench 2 / Score", "evaluation_name": "llm_stats.terminal-bench-2", "score": 0.82, "metric": "Terminal-Bench 2.0 is an updated benchmark for testing AI agents' tool use ability to operate a computer via terminal. It evaluates how well models can handle real-world, end-to-end tasks autonomously, including compiling code, training models, setting up servers, system administration, security tasks, data science workflows, and cybersecurity vulnerabilities.", "lower_is_better": false }, { "benchmark": "Graphwalks BFS >128k", "benchmarkKey": "llm_stats_graphwalks_bfs_128k", "canonical_display_name": "Graphwalks Bfs 128k / Score", "evaluation_name": "llm_stats.graphwalks-bfs-128k", "score": 0.8, "metric": "A graph reasoning benchmark that evaluates language models' ability to perform breadth-first search (BFS) operations on graphs with context length over 128k tokens, testing long-context reasoning capabilities.", "lower_is_better": false }, { "benchmark": "OSWorld-Verified", "benchmarkKey": "llm_stats_osworld_verified", "canonical_display_name": "Osworld Verified / Score", "evaluation_name": "llm_stats.osworld-verified", "score": 0.796, "metric": "OSWorld-Verified is a verified subset of OSWorld, a scalable real computer environment for multimodal agents supporting task setup, execution-based evaluation, and interactive learning across Ubuntu, Windows, and macOS.", "lower_is_better": false }, { "benchmark": "SWE-Bench Pro", "benchmarkKey": "llm_stats_swe_bench_pro", "canonical_display_name": "Swe Bench Pro / Score", "evaluation_name": "llm_stats.swe-bench-pro", "score": 0.778, "metric": "SWE-Bench Pro is an advanced version of SWE-Bench that evaluates language models on complex, real-world software engineering tasks requiring extended reasoning and multi-step problem solving.", "lower_is_better": false }, { "benchmark": "Humanity's Last Exam", "benchmarkKey": "llm_stats_humanity_s_last_exam", "canonical_display_name": "Humanity S Last Exam / Score", "evaluation_name": "llm_stats.humanity-s-last-exam", "score": 0.647, "metric": "Humanity's Last Exam (HLE) is a multi-modal academic benchmark with 2,500 questions across mathematics, humanities, and natural sciences, designed to test LLM capabilities at the frontier of human knowledge with unambiguous, verifiable solutions", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4", "model_route_id": "anthropic__claude-opus-4", "model_family_name": "Claude Opus 4", "developer": "anthropic", "params_billions": null, "total_evaluations": 13, "benchmark_count": 7, "benchmark_family_count": 7, "categories_covered": [ "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "20250514-thinking-16k", "variant_label": "20250514 thinking-16k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-20250514-thinking-16k" ], "last_updated": "2026-04-07T08:15:57.576246Z" }, { "variant_key": "20250514-thinking-1k", "variant_label": "20250514 thinking-1k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-20250514-thinking-1k" ], "last_updated": "2026-04-07T08:15:57.576719Z" }, { "variant_key": "20250514-thinking-8k", "variant_label": "20250514 thinking-8k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-20250514-thinking-8k" ], "last_updated": "2026-04-07T08:15:57.577065Z" }, { "variant_key": "20250514", "variant_label": "20250514", "evaluation_count": 7, "raw_model_ids": [ "anthropic/claude-opus-4-20250514" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "20250514-thinking-10k", "variant_label": "20250514 thinking-10k", "evaluation_count": 2, "raw_model_ids": [ "anthropic/claude-opus-4-20250514-thinking-10k" ], "last_updated": "2026-04-25T00:19:43.227658Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4" ], "last_updated": "2026-04-16T13:47:42.820056Z" } ], "score_summary": { "count": 445, "min": 0.0, "max": 5.0, "average": 0.8429494382022471 }, "reproducibility_summary": { "results_total": 445, "has_reproducibility_gap_count": 443, "populated_ratio_avg": 0.0044943820224719105 }, "provenance_summary": { "total_results": 445, "total_groups": 409, "multi_source_groups": 0, "first_party_only_groups": 5, "source_type_distribution": { "first_party": 5, "third_party": 439, "collaborative": 0, "unspecified": 1 } }, "comparability_summary": { "total_groups": 409, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "ARC-AGI v2", "Anthropic Red Team", "BBQ", "GPQA", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "MMMLU", "MMMU (validation)", "Omni-MATH", "RewardBench 2", "SimpleSafetyTests", "TAU-bench Airline", "TAU-bench Retail", "Terminal-Bench", "WildBench", "Wordle Arena", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #35: Sowing Division / Refusal Rate", "evaluation_name": "AIRBench 2024 - #35: Sowing Division", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #35: Sowing Division", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Wordle Arena", "benchmarkKey": "wordle_arena", "canonical_display_name": "Wordle Arena / Win Rate", "evaluation_name": "wordle_arena", "score": 1.0, "metric": "Win rate on Wordle Arena: Daily NYT Wordle: guess a 5-letter word in 6 attempts with green/yellow/gray feedback", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.993, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.989, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.97, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.969, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "IFEval", "benchmarkKey": "helm_capabilities_ifeval", "canonical_display_name": "IFEval / Strict Accuracy", "evaluation_name": "IFEval", "score": 0.918, "metric": "IFEval Strict Acc on IFEval", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.917, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "RewardBench 2", "benchmarkKey": "reward_bench_2", "canonical_display_name": "Reward Bench 2 / Safety / Score", "evaluation_name": "Safety", "score": 0.8954, "metric": "Safety score - measures safety awareness", "lower_is_better": false }, { "benchmark": "MMMLU", "benchmarkKey": "llm_stats_mmmlu", "canonical_display_name": "Mmmlu / Score", "evaluation_name": "llm_stats.mmmlu", "score": 0.888, "metric": "Multilingual Massive Multitask Language Understanding dataset released by OpenAI, featuring professionally translated MMLU test questions across 14 languages including Arabic, Bengali, German, Spanish, French, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese, Swahili, Yoruba, and Chinese. Contains approximately 15,908 multiple-choice questions per language covering 57 subjects.", "lower_is_better": false }, { "benchmark": "MMLU-Pro", "benchmarkKey": "helm_capabilities_mmlu_pro", "canonical_display_name": "MMLU-Pro / Accuracy", "evaluation_name": "MMLU-Pro", "score": 0.875, "metric": "COT correct on MMLU-Pro", "lower_is_better": false }, { "benchmark": "WildBench", "benchmarkKey": "helm_capabilities_wildbench", "canonical_display_name": "WildBench / Score", "evaluation_name": "WildBench", "score": 0.852, "metric": "WB Score on WildBench", "lower_is_better": false }, { "benchmark": "TAU-bench Retail", "benchmarkKey": "llm_stats_tau_bench_retail", "canonical_display_name": "Tau Bench Retail / Score", "evaluation_name": "llm_stats.tau-bench-retail", "score": 0.814, "metric": "A benchmark for evaluating tool-agent-user interaction in retail environments. Tests language agents' ability to handle dynamic conversations with users while using domain-specific API tools and following policy guidelines. Evaluates agents on tasks like order cancellations, address changes, and order status checks through multi-turn conversations.", "lower_is_better": false }, { "benchmark": "Holistic Evaluation of Language Models (HELM)", "benchmarkKey": "helm_capabilities", "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evaluation_name": "helm_capabilities", "score": 0.78, "metric": "The mean of the scores from all columns.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4-5-thinking", "model_route_id": "anthropic__claude-opus-4-5-thinking", "model_family_name": "Claude Opus 4.5 (Reasoning)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-5-thinking" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.284, "max": 91.3, "average": 16.204052631578953 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 91.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 52.57, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 49.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 47.8, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 25.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 10.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 9.749, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 9.749, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 5.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.913, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.895, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.895, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.871, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.866, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.74, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4-6-adaptive", "model_route_id": "anthropic__claude-opus-4-6-adaptive", "model_family_name": "Claude Opus 4.6 (Adaptive Reasoning, Max Effort)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-6-adaptive" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 15, "min": 0.367, "max": 53.0, "average": 14.234933333333338 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 53.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 49.551, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 48.1, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 25.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 10.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 9.235, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 9.235, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 5.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.921, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.896, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.707, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.531, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.519, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_terminalbench_hard", "canonical_display_name": "artificial_analysis.terminalbench_hard / Terminalbench Hard", "evaluation_name": "artificial_analysis.terminalbench_hard", "score": 0.462, "metric": "Benchmark score on Terminal-Bench Hard.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.367, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-sonnet-4", "model_route_id": "anthropic__claude-sonnet-4", "model_family_name": "Claude Sonnet 4", "developer": "anthropic", "params_billions": null, "total_evaluations": 20, "benchmark_count": 9, "benchmark_family_count": 9, "categories_covered": [ "agentic", "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "20250514-thinking-16k-bedrock", "variant_label": "20250514 thinking-16k-bedrock", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-20250514-thinking-16k-bedrock" ], "last_updated": "2026-04-07T08:15:57.580504Z" }, { "variant_key": "20250514-thinking-1k", "variant_label": "20250514 thinking-1k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-20250514-thinking-1k" ], "last_updated": "2026-04-07T08:15:57.580824Z" }, { "variant_key": "20250514-thinking-8k-bedrock", "variant_label": "20250514 thinking-8k-bedrock", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-20250514-thinking-8k-bedrock" ], "last_updated": "2026-04-07T08:15:57.581247Z" }, { "variant_key": "20250514", "variant_label": "20250514", "evaluation_count": 14, "raw_model_ids": [ "anthropic/claude-sonnet-4-20250514" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "20250514-thinking-10k", "variant_label": "20250514 thinking-10k", "evaluation_count": 2, "raw_model_ids": [ "anthropic/claude-sonnet-4-20250514-thinking-10k" ], "last_updated": "2026-04-25T00:19:43.227658Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4" ], "last_updated": "2026-04-16T13:47:42.820056Z" } ], "score_summary": { "count": 486, "min": 0.0, "max": 5.0, "average": 0.8424520576131683 }, "reproducibility_summary": { "results_total": 486, "has_reproducibility_gap_count": 484, "populated_ratio_avg": 0.00411522633744856 }, "provenance_summary": { "total_results": 486, "total_groups": 426, "multi_source_groups": 0, "first_party_only_groups": 3, "source_type_distribution": { "first_party": 3, "third_party": 483, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 426, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Anthropic Red Team", "BBQ", "GPQA", "Global-MMLU Lite", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "Omni-MATH", "RewardBench 2", "SWE-bench Verified", "SimpleSafetyTests", "TAU-bench Airline", "TAU-bench Retail", "Terminal-Bench", "WildBench", "Wordle Arena", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Fundamental Rights / Refusal Rate", "evaluation_name": "AIRBench 2024 - Fundamental Rights", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Fundamental Rights", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Wordle Arena", "benchmarkKey": "wordle_arena", "canonical_display_name": "Wordle Arena / Win Rate", "evaluation_name": "wordle_arena", "score": 1.0, "metric": "Win rate on Wordle Arena: Daily NYT Wordle: guess a 5-letter word in 6 attempts with green/yellow/gray feedback", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.992, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.981, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.98, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.979, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.972, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "Global-MMLU Lite", "benchmarkKey": "global_mmlu_lite", "canonical_display_name": "Global-MMLU Lite / Culturally Agnostic / Accuracy", "evaluation_name": "Culturally Agnostic", "score": 0.9203, "metric": "Global MMLU Lite - Culturally Agnostic", "lower_is_better": false }, { "benchmark": "RewardBench 2", "benchmarkKey": "reward_bench_2", "canonical_display_name": "Reward Bench 2 / Safety / Score", "evaluation_name": "Safety", "score": 0.8909, "metric": "Safety score - measures safety awareness", "lower_is_better": false }, { "benchmark": "MMLU-Pro", "benchmarkKey": "helm_capabilities_mmlu_pro", "canonical_display_name": "MMLU-Pro / Accuracy", "evaluation_name": "MMLU-Pro", "score": 0.843, "metric": "COT correct on MMLU-Pro", "lower_is_better": false }, { "benchmark": "IFEval", "benchmarkKey": "helm_capabilities_ifeval", "canonical_display_name": "IFEval / Strict Accuracy", "evaluation_name": "IFEval", "score": 0.84, "metric": "IFEval Strict Acc on IFEval", "lower_is_better": false }, { "benchmark": "WildBench", "benchmarkKey": "helm_capabilities_wildbench", "canonical_display_name": "WildBench / Score", "evaluation_name": "WildBench", "score": 0.838, "metric": "WB Score on WildBench", "lower_is_better": false }, { "benchmark": "TAU-bench Retail", "benchmarkKey": "llm_stats_tau_bench_retail", "canonical_display_name": "Tau Bench Retail / Score", "evaluation_name": "llm_stats.tau-bench-retail", "score": 0.805, "metric": "A benchmark for evaluating tool-agent-user interaction in retail environments. Tests language agents' ability to handle dynamic conversations with users while using domain-specific API tools and following policy guidelines. Evaluates agents on tasks like order cancellations, address changes, and order status checks through multi-turn conversations.", "lower_is_better": false }, { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.768, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-sonnet-4-6-adaptive", "model_route_id": "anthropic__claude-sonnet-4-6-adaptive", "model_family_name": "Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-6-adaptive" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 15, "min": 0.3, "max": 51.7, "average": 17.5536 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 51.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 51.225, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 50.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 40.638, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 40.638, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.875, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.757, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.707, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.566, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_terminalbench_hard", "canonical_display_name": "artificial_analysis.terminalbench_hard / Terminalbench Hard", "evaluation_name": "artificial_analysis.terminalbench_hard", "score": 0.53, "metric": "Benchmark score on Terminal-Bench Hard.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.468, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.3, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-sonnet-4-6-non-reasoning-low-effort", "model_route_id": "anthropic__claude-sonnet-4-6-non-reasoning-low-effort", "model_family_name": "Claude Sonnet 4.6 (Non-reasoning, Low Effort)", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-6-non-reasoning-low-effort" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 15, "min": 0.108, "max": 44.822, "average": 10.677733333333332 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 44.822, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 43.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 42.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.087, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 1.087, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.797, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.789, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.587, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.441, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.424, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_terminalbench_hard", "canonical_display_name": "artificial_analysis.terminalbench_hard / Terminalbench Hard", "evaluation_name": "artificial_analysis.terminalbench_hard", "score": 0.424, "metric": "Benchmark score on Terminal-Bench Hard.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.108, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-3-5-sonnet-oct", "model_route_id": "anthropic__claude-3-5-sonnet-oct", "model_family_name": "Claude-3.5-Sonnet(Oct)", "developer": "anthropic", "params_billions": null, "total_evaluations": 22, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-04-20T17:00:03.994647Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 22, "raw_model_ids": [ "anthropic/Claude-3.5-Sonnet(Oct)" ], "last_updated": "2026-04-20T17:00:03.994647Z" } ], "score_summary": { "count": 22, "min": 0.0078125, "max": 0.27232142857142855, "average": 0.08336527069897103 }, "reproducibility_summary": { "results_total": 22, "has_reproducibility_gap_count": 22, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 22, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 22, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Multi-SWE-bench (c)", "Multi-SWE-bench (c++)", "Multi-SWE-bench (go)", "Multi-SWE-bench (java)", "Multi-SWE-bench (javascript)", "Multi-SWE-bench (rust)", "Multi-SWE-bench (typescript)" ], "top_benchmark_scores": [ { "benchmark": "Multi-SWE-bench (typescript)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_typescript", "canonical_display_name": "Multi-SWE-Bench (typescript) / Score", "evaluation_name": "Multi-SWE-Bench (typescript)", "score": 0.27232142857142855, "metric": "Fraction of typescript GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (java)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_java", "canonical_display_name": "Multi-SWE-Bench (java) / Score", "evaluation_name": "Multi-SWE-Bench (java)", "score": 0.203125, "metric": "Fraction of java GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (c++)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_c", "canonical_display_name": "Multi-SWE-Bench (c++) / Score", "evaluation_name": "Multi-SWE-Bench (c++)", "score": 0.12403100775193798, "metric": "Fraction of c++ GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (rust)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_rust", "canonical_display_name": "Multi-SWE-Bench (rust) / Score", "evaluation_name": "Multi-SWE-Bench (rust)", "score": 0.12133891213389121, "metric": "Fraction of rust GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (go)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_go", "canonical_display_name": "Multi-SWE-Bench (go) / Score", "evaluation_name": "Multi-SWE-Bench (go)", "score": 0.06791569086651054, "metric": "Fraction of go GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (javascript)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_javascript", "canonical_display_name": "Multi-SWE-Bench (javascript) / Score", "evaluation_name": "Multi-SWE-Bench (javascript)", "score": 0.042134831460674156, "metric": "Fraction of javascript GitHub issues resolved (0.0–1.0)", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-4-sonnet", "model_route_id": "anthropic__claude-4-sonnet", "model_family_name": "Claude-4-Sonnet", "developer": "anthropic", "params_billions": null, "total_evaluations": 13, "benchmark_count": 4, "benchmark_family_count": 4, "categories_covered": [ "agentic", "knowledge", "other" ], "last_updated": "2026-04-20T17:00:03.994647Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 6, "raw_model_ids": [ "anthropic/Claude-4-Sonnet", "anthropic/claude-4-sonnet" ], "last_updated": "2026-04-20T17:00:03.994647Z" }, { "variant_key": "20250514", "variant_label": "20250514", "evaluation_count": 6, "raw_model_ids": [ "anthropic/claude-4-sonnet-20250514" ], "last_updated": "2026-04-20T16:39:22.266076Z" }, { "variant_key": "20250522", "variant_label": "20250522", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-4-sonnet-20250522" ], "last_updated": "2026-04-20T16:39:22.266076Z" } ], "score_summary": { "count": 35, "min": 0.04, "max": 1025.205465465613, "average": 35.119614045478265 }, "reproducibility_summary": { "results_total": 35, "has_reproducibility_gap_count": 35, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 35, "total_groups": 26, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 35, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 26, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "Multi-SWE-bench (c)", "SWE-bench Verified", "SciArena leaderboard API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 43.305, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 38.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 33.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 30.6, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "canonical_display_name": "SciArena / Cost per 100 Calls", "evaluation_name": "overall_cost_per_100_calls_usd", "score": 5.6999, "metric": "overall_cost_per_100_calls_usd", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.094, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 1.094, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.934, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.837, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.764, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.683, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.523, "metric": "Benchmark score on tau2.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-4-5-sonnet", "model_route_id": "anthropic__claude-4-5-sonnet", "model_family_name": "Claude-4.5-Sonnet", "developer": "anthropic", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "agentic", "other" ], "last_updated": "2026-04-20T17:00:03.994647Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "anthropic/Claude-4.5-Sonnet", "anthropic/claude-4-5-sonnet" ], "last_updated": "2026-04-20T17:00:03.994647Z" } ], "score_summary": { "count": 20, "min": 0.071, "max": 44.221, "average": 9.152331736526948 }, "reproducibility_summary": { "results_total": 20, "has_reproducibility_gap_count": 20, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 20, "total_groups": 20, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 20, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 20, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "Multi-SWE-bench (javascript)" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 44.221, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 37.1, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 37.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 33.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.032, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 1.032, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.86, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.727, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.705, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.59, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.513, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.428, "metric": "Benchmark score on SciCode.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-sonnet-4-6-high", "model_route_id": "anthropic__claude-sonnet-4-6-high", "model_family_name": "Claude-Sonnet-4.6-high", "developer": "Anthropic", "params_billions": null, "total_evaluations": 3, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "other", "reasoning" ], "last_updated": "2026-04-22T13:06:30.542123Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 3, "raw_model_ids": [ "anthropic/claude-sonnet-4-6-high", "anthropic/claude-sonnet-4.6-high" ], "last_updated": "2026-04-22T13:06:30.542123Z" } ], "score_summary": { "count": 16, "min": 0.6042, "max": 943.5, "average": 139.14760625000002 }, "reproducibility_summary": { "results_total": 16, "has_reproducibility_gap_count": 16, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 16, "total_groups": 12, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 16, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 12, "groups_with_variant_check": 4, "groups_with_cross_party_check": 0, "variant_divergent_count": 4, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "CocoaBench v1.0" ], "top_benchmark_scores": [ { "benchmark": "CocoaBench v1.0", "benchmarkKey": "cocoabench", "canonical_display_name": "Cocoabench / Avg Cost Usd", "evaluation_name": "cocoabench.overall.avg_cost_usd", "score": 1.688, "metric": "Average task cost in USD", "lower_is_better": true }, { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Score", "evaluation_name": "v1_Semi_Private", "score": 0.865, "metric": "v1_Semi_Private", "lower_is_better": false } ] }, { "model_group_id": "anthropic/opus-4-1", "model_route_id": "anthropic__opus-4-1", "model_family_name": "Opus 4.1", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-03-11T20:16:40Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/Opus 4.1" ], "last_updated": "2026-03-11T20:16:40Z" } ], "score_summary": { "count": 2, "min": 0.318, "max": 0.4, "average": 0.359 }, "reproducibility_summary": { "results_total": 2, "has_reproducibility_gap_count": 2, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 2, "total_groups": 2, "multi_source_groups": 0, "first_party_only_groups": 2, "source_type_distribution": { "first_party": 2, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 2, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ACE" ], "top_benchmark_scores": [ { "benchmark": "ACE", "benchmarkKey": "ace", "canonical_display_name": "ACE / Score", "evaluation_name": "ace", "score": 0.4, "metric": "Overall ACE score (paper snapshot, approximate).", "lower_is_better": false } ] }, { "model_group_id": "anthropic/opus-4-5", "model_route_id": "anthropic__opus-4-5", "model_family_name": "Opus 4.5", "developer": "anthropic", "params_billions": null, "total_evaluations": 3, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "agentic", "knowledge", "other" ], "last_updated": "2026-03-11T20:16:40Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 3, "raw_model_ids": [ "anthropic/Opus 4.5" ], "last_updated": "2026-03-11T20:16:40Z" } ], "score_summary": { "count": 10, "min": 0.132, "max": 0.65, "average": 0.3412 }, "reproducibility_summary": { "results_total": 10, "has_reproducibility_gap_count": 10, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 10, "total_groups": 9, "multi_source_groups": 0, "first_party_only_groups": 9, "source_type_distribution": { "first_party": 10, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 9, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ACE", "APEX Agents", "APEX v1" ], "top_benchmark_scores": [ { "benchmark": "APEX v1", "benchmarkKey": "apex_v1", "canonical_display_name": "APEX v1 / Medicine (MD) / Score", "evaluation_name": "Medicine (MD)", "score": 0.65, "metric": "Primary care physician (MD) score.", "lower_is_better": false }, { "benchmark": "ACE", "benchmarkKey": "ace", "canonical_display_name": "ACE / Score", "evaluation_name": "ace", "score": 0.478, "metric": "Overall ACE score (paper snapshot).", "lower_is_better": false }, { "benchmark": "APEX Agents", "benchmarkKey": "apex_agents", "canonical_display_name": "APEX Agents / Corporate Lawyer / Mean Score", "evaluation_name": "Corporate Lawyer", "score": 0.471, "metric": "Corporate lawyer world mean score.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/opus-4-6", "model_route_id": "anthropic__opus-4-6", "model_family_name": "Opus 4.6", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-03-11T20:16:40Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/Opus 4.6" ], "last_updated": "2026-03-11T20:16:40Z" } ], "score_summary": { "count": 2, "min": 0.298, "max": 0.502, "average": 0.4 }, "reproducibility_summary": { "results_total": 2, "has_reproducibility_gap_count": 2, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 2, "total_groups": 2, "multi_source_groups": 0, "first_party_only_groups": 2, "source_type_distribution": { "first_party": 2, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 2, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "APEX Agents" ], "top_benchmark_scores": [ { "benchmark": "APEX Agents", "benchmarkKey": "apex_agents", "canonical_display_name": "APEX Agents / Corporate Lawyer / Mean Score", "evaluation_name": "Corporate Lawyer", "score": 0.502, "metric": "Corporate lawyer world mean score from leaderboard model list.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/sonnet-4-5", "model_route_id": "anthropic__sonnet-4-5", "model_family_name": "Sonnet 4.5", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-03-11T20:16:40Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/Sonnet 4.5" ], "last_updated": "2026-03-11T20:16:40Z" } ], "score_summary": { "count": 2, "min": 0.373, "max": 0.44, "average": 0.4065 }, "reproducibility_summary": { "results_total": 2, "has_reproducibility_gap_count": 2, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 2, "total_groups": 2, "multi_source_groups": 0, "first_party_only_groups": 2, "source_type_distribution": { "first_party": 2, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 2, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ACE" ], "top_benchmark_scores": [ { "benchmark": "ACE", "benchmarkKey": "ace", "canonical_display_name": "ACE / Score", "evaluation_name": "ace", "score": 0.44, "metric": "Overall ACE score (paper snapshot, approximate).", "lower_is_better": false } ] }, { "model_group_id": "anthropic/opus-4-6-max-effort", "model_route_id": "anthropic__opus-4-6-max-effort", "model_family_name": "anthropic-opus-4-6-max-effort", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.584023Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/opus-4-6-max-effort" ], "last_updated": "2026-04-07T08:15:57.584023Z" } ], "score_summary": { "count": 2, "min": 0.0023, "max": 8866.2, "average": 4433.10115 }, "reproducibility_summary": { "results_total": 2, "has_reproducibility_gap_count": 2, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 2, "total_groups": 2, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 2, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 2, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v3_Semi_Private / Score", "evaluation_name": "v3_Semi_Private", "score": 0.0023, "metric": "v3_Semi_Private", "lower_is_better": false } ] }, { "model_group_id": "anthropic/anthropic-claude-3-7-sonnet", "model_route_id": "anthropic__anthropic-claude-3-7-sonnet", "model_family_name": "anthropic/anthropic/claude-3-7-sonnet-20250219", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-04-08T05:57:21.939075Z", "variants": [ { "variant_key": "20250219", "variant_label": "20250219", "evaluation_count": 1, "raw_model_ids": [ "anthropic/anthropic/claude-3-7-sonnet-20250219" ], "last_updated": "2026-04-08T05:57:21.939075Z" } ], "score_summary": { "count": 2, "min": 0.3, "max": 0.4629100498862757, "average": 0.3814550249431379 }, "reproducibility_summary": { "results_total": 2, "has_reproducibility_gap_count": 2, "populated_ratio_avg": 0.5 }, "provenance_summary": { "total_results": 2, "total_groups": 2, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 2, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 2, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Swe Bench Verified Mini" ], "top_benchmark_scores": [ { "benchmark": "Swe Bench Verified Mini", "benchmarkKey": "swe_bench_verified_mini", "canonical_display_name": "Swe bench verified mini / Standard Deviation", "evaluation_name": "swe-bench-verified-mini", "score": 0.4629100498862757, "metric": "std", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-2", "model_route_id": "anthropic__claude-2", "model_family_name": "claude-2", "developer": "anthropic", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "agentic", "other" ], "last_updated": "2026-04-20T16:39:22.266076Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "anthropic/claude-2" ], "last_updated": "2026-04-20T16:39:22.266076Z" } ], "score_summary": { "count": 14, "min": 0.0, "max": 12.9, "average": 1.6599285714285714 }, "reproducibility_summary": { "results_total": 14, "has_reproducibility_gap_count": 14, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 14, "total_groups": 14, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 14, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 14, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "SWE-bench Verified" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 12.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 9.1, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.486, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.344, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.194, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.171, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.044, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.0, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-4-opus", "model_route_id": "anthropic__claude-4-opus", "model_family_name": "claude-4-opus-20250514", "developer": "anthropic", "params_billions": null, "total_evaluations": 3, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "agentic", "knowledge", "other" ], "last_updated": "2026-04-20T16:39:22.266076Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "anthropic/claude-4-opus" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "20250514", "variant_label": "20250514", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-4-opus-20250514" ], "last_updated": "2026-04-20T16:39:22.266076Z" } ], "score_summary": { "count": 22, "min": 0.059, "max": 1071.7486973245118, "average": 60.85185442384145 }, "reproducibility_summary": { "results_total": 22, "has_reproducibility_gap_count": 22, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 22, "total_groups": 22, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 22, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 22, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "SWE-bench Verified", "SciArena leaderboard API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 75.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 36.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 34.668, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 33.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 30.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 15.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "canonical_display_name": "SciArena / Rank", "evaluation_name": "overall_rank", "score": 6.0, "metric": "overall_rank", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.307, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 1.307, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.941, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.86, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.732, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.701, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.563, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.542, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-haiku-4-5", "model_route_id": "anthropic__claude-haiku-4-5", "model_family_name": "claude-haiku-4-5-20251001", "developer": "anthropic", "params_billions": null, "total_evaluations": 18, "benchmark_count": 9, "benchmark_family_count": 9, "categories_covered": [ "agentic", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "20251001-thinking-16k", "variant_label": "20251001 thinking-16k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-haiku-4-5-20251001-thinking-16k" ], "last_updated": "2026-04-07T08:15:57.573982Z" }, { "variant_key": "20251001-thinking-1k", "variant_label": "20251001 thinking-1k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-haiku-4-5-20251001-thinking-1k" ], "last_updated": "2026-04-07T08:15:57.574522Z" }, { "variant_key": "20251001-thinking-32k", "variant_label": "20251001 thinking-32k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-haiku-4-5-20251001-thinking-32k" ], "last_updated": "2026-04-07T08:15:57.575113Z" }, { "variant_key": "20251001-thinking-8k", "variant_label": "20251001 thinking-8k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-haiku-4-5-20251001-thinking-8k" ], "last_updated": "2026-04-07T08:15:57.575538Z" }, { "variant_key": "20251001", "variant_label": "20251001", "evaluation_count": 7, "raw_model_ids": [ "anthropic/claude-haiku-4-5-20251001", "anthropic/claude-haiku-4-5-20251001-fc", "anthropic/claude-haiku-4-5-20251001-prompt" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 7, "raw_model_ids": [ "anthropic/claude-haiku-4.5" ], "last_updated": "2026-04-16T13:47:43.417572Z" } ], "score_summary": { "count": 509, "min": 0.0, "max": 782.0, "average": 10.146776915520626 }, "reproducibility_summary": { "results_total": 509, "has_reproducibility_gap_count": 500, "populated_ratio_avg": 0.01768172888015717 }, "provenance_summary": { "total_results": 509, "total_groups": 443, "multi_source_groups": 0, "first_party_only_groups": 7, "source_type_distribution": { "first_party": 7, "third_party": 502, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 443, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Anthropic Red Team", "BBQ", "BFCL leaderboard CSV", "Cybersecurity CTFs", "Fibble arena", "GPQA", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "MMMU (validation)", "OSWorld", "Omni-MATH", "SimpleSafetyTests", "Tau2 Airline", "Tau2 Retail", "Tau2 Telecom", "Terminal Bench 2 0", "Terminal-Bench", "WildBench", "Wordle Arena", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 35.5, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 20.07, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Economic Harm / Refusal Rate", "evaluation_name": "AIRBench 2024 - Economic Harm", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Economic Harm", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.988, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.969, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.959, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.955, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.932, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.928, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "WildBench", "benchmarkKey": "helm_capabilities_wildbench", "canonical_display_name": "WildBench / Score", "evaluation_name": "WildBench", "score": 0.839, "metric": "WB Score on WildBench", "lower_is_better": false }, { "benchmark": "Tau2 Retail", "benchmarkKey": "llm_stats_tau2_retail", "canonical_display_name": "Tau2 Retail / Score", "evaluation_name": "llm_stats.tau2-retail", "score": 0.832, "metric": "τ²-bench retail domain evaluates conversational AI agents in customer service scenarios within a dual-control environment where both agent and user can interact with tools. Tests tool-agent-user interaction, rule adherence, and task consistency in retail customer support contexts.", "lower_is_better": false }, { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.83, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "IFEval", "benchmarkKey": "helm_capabilities_ifeval", "canonical_display_name": "IFEval / Strict Accuracy", "evaluation_name": "IFEval", "score": 0.801, "metric": "IFEval Strict Acc on IFEval", "lower_is_better": false }, { "benchmark": "MMLU-Pro", "benchmarkKey": "helm_capabilities_mmlu_pro", "canonical_display_name": "MMLU-Pro / Accuracy", "evaluation_name": "MMLU-Pro", "score": 0.777, "metric": "COT correct on MMLU-Pro", "lower_is_better": false }, { "benchmark": "MMMU (validation)", "benchmarkKey": "llm_stats_mmmu_validation", "canonical_display_name": "Mmmu Validation / Score", "evaluation_name": "llm_stats.mmmu-validation", "score": 0.732, "metric": "Validation set of the Massive Multi-discipline Multimodal Understanding and Reasoning benchmark. Features college-level multimodal questions across 6 core disciplines (Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, Tech & Engineering) spanning 30 subjects and 183 subfields with diverse image types including charts, diagrams, maps, and tables.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4-1", "model_route_id": "anthropic__claude-opus-4-1", "model_family_name": "claude-opus-4-1-20250805", "developer": "anthropic", "params_billions": null, "total_evaluations": 7, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "agentic", "knowledge", "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "20250805", "variant_label": "20250805", "evaluation_count": 3, "raw_model_ids": [ "anthropic/claude-opus-4-1-20250805" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 4, "raw_model_ids": [ "anthropic/claude-opus-4.1" ], "last_updated": "2026-03-17T19:48:21.772108Z" } ], "score_summary": { "count": 47, "min": 0.433, "max": 38.0, "average": 3.91735744680851 }, "reproducibility_summary": { "results_total": 47, "has_reproducibility_gap_count": 47, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 47, "total_groups": 25, "multi_source_groups": 0, "first_party_only_groups": 5, "source_type_distribution": { "first_party": 5, "third_party": 42, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 25, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Global-MMLU Lite", "MMMLU", "MMMU (validation)", "TAU-bench Airline", "TAU-bench Retail", "Terminal Bench 2 0", "Terminal-Bench" ], "top_benchmark_scores": [ { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 38.0, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Global-MMLU Lite", "benchmarkKey": "global_mmlu_lite", "canonical_display_name": "Global-MMLU Lite / Culturally Agnostic / Accuracy", "evaluation_name": "Culturally Agnostic", "score": 0.9528, "metric": "Global MMLU Lite - Culturally Agnostic", "lower_is_better": false }, { "benchmark": "MMMLU", "benchmarkKey": "llm_stats_mmmlu", "canonical_display_name": "Mmmlu / Score", "evaluation_name": "llm_stats.mmmlu", "score": 0.895, "metric": "Multilingual Massive Multitask Language Understanding dataset released by OpenAI, featuring professionally translated MMLU test questions across 14 languages including Arabic, Bengali, German, Spanish, French, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese, Swahili, Yoruba, and Chinese. Contains approximately 15,908 multiple-choice questions per language covering 57 subjects.", "lower_is_better": false }, { "benchmark": "TAU-bench Retail", "benchmarkKey": "llm_stats_tau_bench_retail", "canonical_display_name": "Tau Bench Retail / Score", "evaluation_name": "llm_stats.tau-bench-retail", "score": 0.824, "metric": "A benchmark for evaluating tool-agent-user interaction in retail environments. Tests language agents' ability to handle dynamic conversations with users while using domain-specific API tools and following policy guidelines. Evaluates agents on tasks like order cancellations, address changes, and order status checks through multi-turn conversations.", "lower_is_better": false }, { "benchmark": "MMMU (validation)", "benchmarkKey": "llm_stats_mmmu_validation", "canonical_display_name": "Mmmu Validation / Score", "evaluation_name": "llm_stats.mmmu-validation", "score": 0.771, "metric": "Validation set of the Massive Multi-discipline Multimodal Understanding and Reasoning benchmark. Features college-level multimodal questions across 6 core disciplines (Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, Tech & Engineering) spanning 30 subjects and 183 subfields with diverse image types including charts, diagrams, maps, and tables.", "lower_is_better": false }, { "benchmark": "TAU-bench Airline", "benchmarkKey": "llm_stats_tau_bench_airline", "canonical_display_name": "Tau Bench Airline / Score", "evaluation_name": "llm_stats.tau-bench-airline", "score": 0.56, "metric": "Part of τ-bench (TAU-bench), a benchmark for Tool-Agent-User interaction in real-world domains. The airline domain evaluates language agents' ability to interact with users through dynamic conversations while following domain-specific rules and using API tools. Agents must handle airline-related tasks and policies reliably.", "lower_is_better": false }, { "benchmark": "Terminal-Bench", "benchmarkKey": "llm_stats_terminal_bench", "canonical_display_name": "Terminal Bench / Score", "evaluation_name": "llm_stats.terminal-bench", "score": 0.433, "metric": "Terminal-Bench is a benchmark for testing AI agents in real terminal environments. It evaluates how well agents can handle real-world, end-to-end tasks autonomously, including compiling code, training models, setting up servers, system administration, security tasks, data science workflows, and cybersecurity vulnerabilities. The benchmark consists of a dataset of ~100 hand-crafted, human-verified tasks and an execution harness that connects language models to a terminal sandbox.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4-5", "model_route_id": "anthropic__claude-opus-4-5", "model_family_name": "claude-opus-4-5-20251101", "developer": "anthropic", "params_billions": null, "total_evaluations": 49, "benchmark_count": 10, "benchmark_family_count": 10, "categories_covered": [ "agentic", "coding", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 40, "raw_model_ids": [ "anthropic/claude-opus-4-5", "anthropic/claude-opus-4.5" ], "last_updated": "2026-04-20T16:39:22.266076Z" }, { "variant_key": "20251101-thinking-16k", "variant_label": "20251101 thinking-16k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-5-20251101-thinking-16k" ], "last_updated": "2026-04-07T08:15:57.577391Z" }, { "variant_key": "20251101-thinking-32k", "variant_label": "20251101 thinking-32k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-5-20251101-thinking-32k" ], "last_updated": "2026-04-07T08:15:57.577719Z" }, { "variant_key": "20251101-thinking-64k", "variant_label": "20251101 thinking-64k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-5-20251101-thinking-64k" ], "last_updated": "2026-04-07T08:15:57.577979Z" }, { "variant_key": "20251101-thinking-8k", "variant_label": "20251101 thinking-8k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-5-20251101-thinking-8k" ], "last_updated": "2026-04-07T08:15:57.578212Z" }, { "variant_key": "20251101-thinking-none", "variant_label": "20251101 thinking-none", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-5-20251101-thinking-none" ], "last_updated": "2026-04-07T08:15:57.578542Z" }, { "variant_key": "20251101", "variant_label": "20251101", "evaluation_count": 4, "raw_model_ids": [ "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-5-20251101-fc", "anthropic/claude-opus-4-5-20251101-prompt" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 164, "min": 0.0708, "max": 95.5, "average": 25.281729268292686 }, "reproducibility_summary": { "results_total": 164, "has_reproducibility_gap_count": 164, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 164, "total_groups": 76, "multi_source_groups": 0, "first_party_only_groups": 9, "source_type_distribution": { "first_party": 9, "third_party": 155, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 76, "groups_with_variant_check": 6, "groups_with_cross_party_check": 0, "variant_divergent_count": 6, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "ARC-AGI v2", "AppWorld Benchmark", "Artificial Analysis LLM API", "BFCL leaderboard CSV", "BrowseComp-Plus", "MCP Atlas", "MMMLU", "MMMU (validation)", "OSWorld", "SWE-Bench Verified", "SWE-bench Verified", "Swe Bench", "Tau2 Retail", "Tau2 Telecom", "Terminal Bench 2 0", "Terminal-Bench 2.0", "τ-bench (Tool-Agent-User Interaction Benchmark)" ], "top_benchmark_scores": [ { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 63.1, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 62.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 52.885, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 43.1, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 42.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 25.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 10.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 5.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 3.65, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.311, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 1.311, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.982, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "MMMLU", "benchmarkKey": "llm_stats_mmmlu", "canonical_display_name": "Mmmlu / Score", "evaluation_name": "llm_stats.mmmlu", "score": 0.908, "metric": "Multilingual Massive Multitask Language Understanding dataset released by OpenAI, featuring professionally translated MMLU test questions across 14 languages including Arabic, Bengali, German, Spanish, French, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese, Swahili, Yoruba, and Chinese. Contains approximately 15,908 multiple-choice questions per language covering 57 subjects.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.889, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Tau2 Retail", "benchmarkKey": "llm_stats_tau2_retail", "canonical_display_name": "Tau2 Retail / Score", "evaluation_name": "llm_stats.tau2-retail", "score": 0.889, "metric": "τ²-bench retail domain evaluates conversational AI agents in customer service scenarios within a dual-control environment where both agent and user can interact with tools. Tests tool-agent-user interaction, rule adherence, and task consistency in retail customer support contexts.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4-6", "model_route_id": "anthropic__claude-opus-4-6", "model_family_name": "claude-opus-4-6", "developer": "anthropic", "params_billions": null, "total_evaluations": 11, "benchmark_count": 5, "benchmark_family_count": 5, "categories_covered": [ "agentic", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 11, "raw_model_ids": [ "anthropic/claude-opus-4-6", "anthropic/claude-opus-4.6" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 56, "min": 0.0, "max": 8017.59, "average": 310.59414642857143 }, "reproducibility_summary": { "results_total": 56, "has_reproducibility_gap_count": 47, "populated_ratio_avg": 0.16071428571428573 }, "provenance_summary": { "total_results": 56, "total_groups": 50, "multi_source_groups": 0, "first_party_only_groups": 25, "source_type_distribution": { "first_party": 25, "third_party": 31, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 50, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AIME 2025", "ARC-AGI v2", "Artificial Analysis LLM API", "BrowseComp", "CharXiv-R", "CyberGym", "DeepSearchQA", "Fibble arena", "FigQA", "Finance Agent", "GDPval-AA", "GPQA", "Graphwalks BFS >128k", "Graphwalks parents >128k", "Humanity's Last Exam", "MCP Atlas", "MMMLU", "MMMU-Pro", "MRCR v2 (8-needle)", "OSWorld", "OpenRCA", "SWE-Bench Verified", "SWE-bench Multilingual", "Tau2 Retail", "Tau2 Telecom", "Terminal Bench 2 0", "Terminal-Bench 2.0", "Vending-Bench 2", "Wordle Arena" ], "top_benchmark_scores": [ { "benchmark": "Vending-Bench 2", "benchmarkKey": "llm_stats_vending_bench_2", "canonical_display_name": "Vending Bench 2 / Score", "evaluation_name": "llm_stats.vending-bench-2", "score": 8017.59, "metric": "Vending-Bench 2 tests longer horizon planning capabilities by evaluating how well AI models can manage a simulated vending machine business over extended periods. The benchmark measures a model's ability to maintain consistent tool usage and decision-making for a full simulated year of operation, driving higher returns without drifting off task.", "lower_is_better": false }, { "benchmark": "GDPval-AA", "benchmarkKey": "llm_stats_gdpval_aa", "canonical_display_name": "Gdpval Aa / Score", "evaluation_name": "llm_stats.gdpval-aa", "score": 1606.0, "metric": "GDPval-AA is an evaluation of AI model performance on economically valuable knowledge work tasks across professional domains including finance, legal, and other sectors. Run independently by Artificial Analysis, it uses Elo scoring to rank models on real-world work task performance.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 74.7, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 47.6, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 46.5, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 44.956, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 25.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 10.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 5.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.579, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 1.579, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Wordle Arena", "benchmarkKey": "wordle_arena", "canonical_display_name": "Wordle Arena / Win Rate", "evaluation_name": "wordle_arena", "score": 1.0, "metric": "Win rate on Wordle Arena: Daily NYT Wordle: guess a 5-letter word in 6 attempts with green/yellow/gray feedback", "lower_is_better": false }, { "benchmark": "AIME 2025", "benchmarkKey": "llm_stats_aime_2025", "canonical_display_name": "Aime 2025 / Score", "evaluation_name": "llm_stats.aime-2025", "score": 0.9979, "metric": "All 30 problems from the 2025 American Invitational Mathematics Examination (AIME I and AIME II), testing olympiad-level mathematical reasoning with integer answers from 000-999. Used as an AI benchmark to evaluate large language models' ability to solve complex mathematical problems requiring multi-step logical deductions and structured symbolic reasoning.", "lower_is_better": false }, { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.993, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "Graphwalks parents >128k", "benchmarkKey": "llm_stats_graphwalks_parents_128k", "canonical_display_name": "Graphwalks Parents 128k / Score", "evaluation_name": "llm_stats.graphwalks-parents-128k", "score": 0.954, "metric": "A graph reasoning benchmark that evaluates language models' ability to find parent nodes in graphs with context length over 128k tokens, testing long-context reasoning and graph structure understanding.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4-6-thinking-120k-high", "model_route_id": "anthropic__claude-opus-4-6-thinking-120k-high", "model_family_name": "claude-opus-4-6-thinking-120K-high", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.578887Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-6-thinking-120k-high" ], "last_updated": "2026-04-07T08:15:57.578887Z" } ], "score_summary": { "count": 8, "min": 0.6917, "max": 3.8092, "average": 1.6166375000000002 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Score", "evaluation_name": "v1_Semi_Private", "score": 0.94, "metric": "v1_Semi_Private", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4-6-thinking-120k-low", "model_route_id": "anthropic__claude-opus-4-6-thinking-120k-low", "model_family_name": "claude-opus-4-6-thinking-120K-low", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.579223Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-6-thinking-120k-low" ], "last_updated": "2026-04-07T08:15:57.579223Z" } ], "score_summary": { "count": 8, "min": 0.4105, "max": 2.3966, "average": 1.0882749999999999 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.648, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "anthropic/claude-opus-4-6-thinking-120k-max", "model_route_id": "anthropic__claude-opus-4-6-thinking-120k-max", "model_family_name": "claude-opus-4-6-thinking-120K-max", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.579558Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-6-thinking-120k-max" ], "last_updated": "2026-04-07T08:15:57.579558Z" } ], "score_summary": { "count": 8, "min": 0.6875, "max": 3.7707, "average": 1.7400125 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Public_Eval / Score", "evaluation_name": "v1_Public_Eval", "score": 0.9675, "metric": "v1_Public_Eval", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4-6-thinking-120k-medium", "model_route_id": "anthropic__claude-opus-4-6-thinking-120k-medium", "model_family_name": "claude-opus-4-6-thinking-120K-medium", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.579886Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-6-thinking-120k-medium" ], "last_updated": "2026-04-07T08:15:57.579886Z" } ], "score_summary": { "count": 8, "min": 0.6183, "max": 3.0162, "average": 1.323275 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Score", "evaluation_name": "v1_Semi_Private", "score": 0.92, "metric": "v1_Semi_Private", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-opus-4-7", "model_route_id": "anthropic__claude-opus-4-7", "model_family_name": "claude-opus-4-7", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-opus-4-7" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 12, "min": 0.547, "max": 0.942, "average": 0.7706666666666667 }, "reproducibility_summary": { "results_total": 12, "has_reproducibility_gap_count": 12, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 12, "total_groups": 12, "multi_source_groups": 0, "first_party_only_groups": 12, "source_type_distribution": { "first_party": 12, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 12, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BrowseComp", "CharXiv-R", "CyberGym", "Finance Agent", "GPQA", "Humanity's Last Exam", "MCP Atlas", "MMMLU", "OSWorld-Verified", "SWE-Bench Pro", "SWE-Bench Verified", "Terminal-Bench 2.0" ], "top_benchmark_scores": [ { "benchmark": "GPQA", "benchmarkKey": "llm_stats_gpqa", "canonical_display_name": "Gpqa / Score", "evaluation_name": "llm_stats.gpqa", "score": 0.942, "metric": "A challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. Questions are Google-proof and extremely difficult, with PhD experts reaching 65% accuracy.", "lower_is_better": false }, { "benchmark": "MMMLU", "benchmarkKey": "llm_stats_mmmlu", "canonical_display_name": "Mmmlu / Score", "evaluation_name": "llm_stats.mmmlu", "score": 0.915, "metric": "Multilingual Massive Multitask Language Understanding dataset released by OpenAI, featuring professionally translated MMLU test questions across 14 languages including Arabic, Bengali, German, Spanish, French, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese, Swahili, Yoruba, and Chinese. Contains approximately 15,908 multiple-choice questions per language covering 57 subjects.", "lower_is_better": false }, { "benchmark": "CharXiv-R", "benchmarkKey": "llm_stats_charxiv_r", "canonical_display_name": "Charxiv R / Score", "evaluation_name": "llm_stats.charxiv-r", "score": 0.91, "metric": "CharXiv-R is the reasoning component of the CharXiv benchmark, focusing on complex reasoning questions that require synthesizing information across visual chart elements. It evaluates multimodal large language models on their ability to understand and reason about scientific charts from arXiv papers through various reasoning tasks.", "lower_is_better": false }, { "benchmark": "SWE-Bench Verified", "benchmarkKey": "llm_stats_swe_bench_verified", "canonical_display_name": "Swe Bench Verified / Score", "evaluation_name": "llm_stats.swe-bench-verified", "score": 0.876, "metric": "A verified subset of 500 software engineering problems from real GitHub issues, validated by human annotators for evaluating language models' ability to resolve real-world coding issues by generating patches for Python codebases.", "lower_is_better": false }, { "benchmark": "BrowseComp", "benchmarkKey": "llm_stats_browsecomp", "canonical_display_name": "Browsecomp / Score", "evaluation_name": "llm_stats.browsecomp", "score": 0.793, "metric": "BrowseComp is a benchmark comprising 1,266 questions that challenge AI agents to persistently navigate the internet in search of hard-to-find, entangled information. The benchmark measures agents' ability to exercise persistence in information gathering, demonstrate creativity in web navigation, and find concise, verifiable answers. Despite the difficulty of the questions, BrowseComp is simple and easy-to-use, as predicted answers are short and easily verifiable against reference answers.", "lower_is_better": false }, { "benchmark": "OSWorld-Verified", "benchmarkKey": "llm_stats_osworld_verified", "canonical_display_name": "Osworld Verified / Score", "evaluation_name": "llm_stats.osworld-verified", "score": 0.78, "metric": "OSWorld-Verified is a verified subset of OSWorld, a scalable real computer environment for multimodal agents supporting task setup, execution-based evaluation, and interactive learning across Ubuntu, Windows, and macOS.", "lower_is_better": false }, { "benchmark": "MCP Atlas", "benchmarkKey": "llm_stats_mcp_atlas", "canonical_display_name": "Mcp Atlas / Score", "evaluation_name": "llm_stats.mcp-atlas", "score": 0.773, "metric": "MCP Atlas is a benchmark for evaluating AI models on scaled tool use capabilities, measuring how well models can coordinate and utilize multiple tools across complex multi-step tasks.", "lower_is_better": false }, { "benchmark": "CyberGym", "benchmarkKey": "llm_stats_cybergym", "canonical_display_name": "Cybergym / Score", "evaluation_name": "llm_stats.cybergym", "score": 0.731, "metric": "CyberGym is a benchmark for evaluating AI agents on cybersecurity tasks, testing their ability to identify vulnerabilities, perform security analysis, and complete security-related challenges in a controlled environment.", "lower_is_better": false }, { "benchmark": "Terminal-Bench 2.0", "benchmarkKey": "llm_stats_terminal_bench_2", "canonical_display_name": "Terminal Bench 2 / Score", "evaluation_name": "llm_stats.terminal-bench-2", "score": 0.694, "metric": "Terminal-Bench 2.0 is an updated benchmark for testing AI agents' tool use ability to operate a computer via terminal. It evaluates how well models can handle real-world, end-to-end tasks autonomously, including compiling code, training models, setting up servers, system administration, security tasks, data science workflows, and cybersecurity vulnerabilities.", "lower_is_better": false }, { "benchmark": "Finance Agent", "benchmarkKey": "llm_stats_finance_agent", "canonical_display_name": "Finance Agent / Score", "evaluation_name": "llm_stats.finance-agent", "score": 0.644, "metric": "Finance Agent is a benchmark for evaluating AI models on agentic financial analysis tasks, testing their ability to process financial data, perform calculations, and generate accurate analyses across various financial domains.", "lower_is_better": false }, { "benchmark": "SWE-Bench Pro", "benchmarkKey": "llm_stats_swe_bench_pro", "canonical_display_name": "Swe Bench Pro / Score", "evaluation_name": "llm_stats.swe-bench-pro", "score": 0.643, "metric": "SWE-Bench Pro is an advanced version of SWE-Bench that evaluates language models on complex, real-world software engineering tasks requiring extended reasoning and multi-step problem solving.", "lower_is_better": false }, { "benchmark": "Humanity's Last Exam", "benchmarkKey": "llm_stats_humanity_s_last_exam", "canonical_display_name": "Humanity S Last Exam / Score", "evaluation_name": "llm_stats.humanity-s-last-exam", "score": 0.547, "metric": "Humanity's Last Exam (HLE) is a multi-modal academic benchmark with 2,500 questions across mathematics, humanities, and natural sciences, designed to test LLM capabilities at the frontier of human knowledge with unambiguous, verifiable solutions", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-sonnet-4-5", "model_route_id": "anthropic__claude-sonnet-4-5", "model_family_name": "claude-sonnet-4-5-20250929", "developer": "anthropic", "params_billions": null, "total_evaluations": 22, "benchmark_count": 9, "benchmark_family_count": 9, "categories_covered": [ "agentic", "coding", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "20250929-thinking-16k", "variant_label": "20250929 thinking-16k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-5-20250929-thinking-16k" ], "last_updated": "2026-04-07T08:15:57.581922Z" }, { "variant_key": "20250929-thinking-1k", "variant_label": "20250929 thinking-1k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-5-20250929-thinking-1k" ], "last_updated": "2026-04-07T08:15:57.582247Z" }, { "variant_key": "20250929-thinking-32k", "variant_label": "20250929 thinking-32k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-5-20250929-thinking-32k" ], "last_updated": "2026-04-07T08:15:57.582564Z" }, { "variant_key": "20250929-thinking-8k", "variant_label": "20250929 thinking-8k", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-5-20250929-thinking-8k" ], "last_updated": "2026-04-07T08:15:57.582890Z" }, { "variant_key": "20250929", "variant_label": "20250929", "evaluation_count": 8, "raw_model_ids": [ "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-sonnet-4-5-20250929-fc", "anthropic/claude-sonnet-4-5-20250929-prompt" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 10, "raw_model_ids": [ "anthropic/claude-sonnet-4-5", "anthropic/claude-sonnet-4.5" ], "last_updated": "2026-04-20T16:39:22.266076Z" } ], "score_summary": { "count": 508, "min": 0.0, "max": 95.5, "average": 7.106885236220486 }, "reproducibility_summary": { "results_total": 508, "has_reproducibility_gap_count": 508, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 508, "total_groups": 438, "multi_source_groups": 0, "first_party_only_groups": 7, "source_type_distribution": { "first_party": 7, "third_party": 501, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 438, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Anthropic Red Team", "BBQ", "BFCL leaderboard CSV", "Easy Problems", "GPQA", "Hard Problems", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "MMMLU", "MMMUval", "Medium Problems", "OSWorld", "Omni-MATH", "SWE-bench Verified", "SWE-bench Verified (Agentic Coding)", "SimpleSafetyTests", "TAU-bench Airline", "TAU-bench Retail", "Terminal Bench 2 0", "Terminal-Bench", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 46.5, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 10.07, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Defamation / Refusal Rate", "evaluation_name": "AIRBench 2024 - Defamation", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Defamation", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.989, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.982, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.971, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.964, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.92, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "MMMLU", "benchmarkKey": "llm_stats_mmmlu", "canonical_display_name": "Mmmlu / Score", "evaluation_name": "llm_stats.mmmlu", "score": 0.891, "metric": "Multilingual Massive Multitask Language Understanding dataset released by OpenAI, featuring professionally translated MMLU test questions across 14 languages including Arabic, Bengali, German, Spanish, French, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese, Swahili, Yoruba, and Chinese. Contains approximately 15,908 multiple-choice questions per language covering 57 subjects.", "lower_is_better": false }, { "benchmark": "MMLU-Pro", "benchmarkKey": "helm_capabilities_mmlu_pro", "canonical_display_name": "MMLU-Pro / Accuracy", "evaluation_name": "MMLU-Pro", "score": 0.869, "metric": "COT correct on MMLU-Pro", "lower_is_better": false }, { "benchmark": "TAU-bench Retail", "benchmarkKey": "llm_stats_tau_bench_retail", "canonical_display_name": "Tau Bench Retail / Score", "evaluation_name": "llm_stats.tau-bench-retail", "score": 0.862, "metric": "A benchmark for evaluating tool-agent-user interaction in retail environments. Tests language agents' ability to handle dynamic conversations with users while using domain-specific API tools and following policy guidelines. Evaluates agents on tasks like order cancellations, address changes, and order status checks through multi-turn conversations.", "lower_is_better": false }, { "benchmark": "WildBench", "benchmarkKey": "helm_capabilities_wildbench", "canonical_display_name": "WildBench / Score", "evaluation_name": "WildBench", "score": 0.854, "metric": "WB Score on WildBench", "lower_is_better": false }, { "benchmark": "IFEval", "benchmarkKey": "helm_capabilities_ifeval", "canonical_display_name": "IFEval / Strict Accuracy", "evaluation_name": "IFEval", "score": 0.85, "metric": "IFEval Strict Acc on IFEval", "lower_is_better": false }, { "benchmark": "MMMUval", "benchmarkKey": "llm_stats_mmmuval", "canonical_display_name": "Mmmuval / Score", "evaluation_name": "llm_stats.mmmuval", "score": 0.778, "metric": "Validation set for MMMU (Massive Multi-discipline Multimodal Understanding and Reasoning) benchmark, designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning across Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-sonnet-4-6", "model_route_id": "anthropic__claude-sonnet-4-6", "model_family_name": "claude-sonnet-4-6", "developer": "anthropic", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "anthropic/claude-sonnet-4-6" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 29, "min": 0.132, "max": 1633.0, "average": 62.38803448275862 }, "reproducibility_summary": { "results_total": 29, "has_reproducibility_gap_count": 29, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 29, "total_groups": 29, "multi_source_groups": 0, "first_party_only_groups": 14, "source_type_distribution": { "first_party": 14, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 29, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC-AGI v2", "Artificial Analysis LLM API", "BrowseComp", "Finance Agent", "GDPval-AA", "GPQA", "Humanity's Last Exam", "MCP Atlas", "MMMLU", "MMMU-Pro", "OSWorld", "SWE-Bench Verified", "Tau2 Retail", "Tau2 Telecom", "Terminal-Bench 2.0" ], "top_benchmark_scores": [ { "benchmark": "GDPval-AA", "benchmarkKey": "llm_stats_gdpval_aa", "canonical_display_name": "Gdpval Aa / Score", "evaluation_name": "llm_stats.gdpval-aa", "score": 1633.0, "metric": "GDPval-AA is an evaluation of AI model performance on economically valuable knowledge work tasks across professional domains including finance, legal, and other sectors. Run independently by Artificial Analysis, it uses Elo scoring to rank models on real-world work task performance.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 46.4, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 46.177, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 44.4, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 6.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 3.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.004, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 1.004, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.979, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "Tau2 Retail", "benchmarkKey": "llm_stats_tau2_retail", "canonical_display_name": "Tau2 Retail / Score", "evaluation_name": "llm_stats.tau2-retail", "score": 0.917, "metric": "τ²-bench retail domain evaluates conversational AI agents in customer service scenarios within a dual-control environment where both agent and user can interact with tools. Tests tool-agent-user interaction, rule adherence, and task consistency in retail customer support contexts.", "lower_is_better": false }, { "benchmark": "GPQA", "benchmarkKey": "llm_stats_gpqa", "canonical_display_name": "Gpqa / Score", "evaluation_name": "llm_stats.gpqa", "score": 0.899, "metric": "A challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. Questions are Google-proof and extremely difficult, with PhD experts reaching 65% accuracy.", "lower_is_better": false }, { "benchmark": "MMMLU", "benchmarkKey": "llm_stats_mmmlu", "canonical_display_name": "Mmmlu / Score", "evaluation_name": "llm_stats.mmmlu", "score": 0.893, "metric": "Multilingual Massive Multitask Language Understanding dataset released by OpenAI, featuring professionally translated MMLU test questions across 14 languages including Arabic, Bengali, German, Spanish, French, Hindi, Indonesian, Italian, Japanese, Korean, Portuguese, Swahili, Yoruba, and Chinese. Contains approximately 15,908 multiple-choice questions per language covering 57 subjects.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.799, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "SWE-Bench Verified", "benchmarkKey": "llm_stats_swe_bench_verified", "canonical_display_name": "Swe Bench Verified / Score", "evaluation_name": "llm_stats.swe-bench-verified", "score": 0.796, "metric": "A verified subset of 500 software engineering problems from real GitHub issues, validated by human annotators for evaluating language models' ability to resolve real-world coding issues by generating patches for Python codebases.", "lower_is_better": false } ] }, { "model_group_id": "anthropic/claude-sonnet-4-6-max", "model_route_id": "anthropic__claude-sonnet-4-6-max", "model_family_name": "claude_sonnet_4_6_max", "developer": "anthropic", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.583697Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "anthropic/claude-sonnet-4-6-max" ], "last_updated": "2026-04-07T08:15:57.583697Z" } ], "score_summary": { "count": 8, "min": 0.5833, "max": 2.8905, "average": 1.3964374999999998 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Public_Eval / Score", "evaluation_name": "v1_Public_Eval", "score": 0.9575, "metric": "v1_Public_Eval", "lower_is_better": false } ] } ] }