{ "developer": "OpenAI", "models": [ { "model_group_id": "openai/chatgpt-agent", "model_route_id": "openai__chatgpt-agent", "model_family_name": "ChatGPT Agent", "developer": "OpenAI", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-22T13:06:30.542123Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/chatgpt-agent" ], "last_updated": "2026-04-22T13:06:30.542123Z" } ], "score_summary": { "count": 1, "min": 26.8, "max": 26.8, "average": 26.8 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 1, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "CocoaBench v1.0" ], "top_benchmark_scores": [ { "benchmark": "CocoaBench v1.0", "benchmarkKey": "cocoabench", "canonical_display_name": "Cocoabench / Accuracy Percent", "evaluation_name": "cocoabench.overall.accuracy_percent", "score": 26.8, "metric": "Overall task success rate on CocoaBench aggregate release", "lower_is_better": false } ] }, { "model_group_id": "openai/codex", "model_route_id": "openai__codex", "model_family_name": "CodeX", "developer": "OpenAI", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-22T13:06:30.542123Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/codex" ], "last_updated": "2026-04-22T13:06:30.542123Z" } ], "score_summary": { "count": 4, "min": 0.747, "max": 377.8, "average": 133.75225 }, "reproducibility_summary": { "results_total": 4, "has_reproducibility_gap_count": 4, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 4, "total_groups": 4, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 4, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 4, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "CocoaBench v1.0" ], "top_benchmark_scores": [ { "benchmark": "CocoaBench v1.0", "benchmarkKey": "cocoabench", "canonical_display_name": "Cocoabench / Avg Cost Usd", "evaluation_name": "cocoabench.overall.avg_cost_usd", "score": 0.747, "metric": "Average task cost in USD", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-oss-20b-high", "model_route_id": "openai__gpt-oss-20b-high", "model_family_name": "GPT OSS 20B High", "developer": "openai", "params_billions": 20.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-oss-20b-high" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 1, "min": 0.987, "max": 0.987, "average": 0.987 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 1, "source_type_distribution": { "first_party": 1, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AIME 2025" ], "top_benchmark_scores": [ { "benchmark": "AIME 2025", "benchmarkKey": "llm_stats_aime_2025", "canonical_display_name": "Aime 2025 / Score", "evaluation_name": "llm_stats.aime-2025", "score": 0.987, "metric": "All 30 problems from the 2025 American Invitational Mathematics Examination (AIME I and AIME II), testing olympiad-level mathematical reasoning with integer answers from 000-999. Used as an AI benchmark to evaluate large language models' ability to solve complex mathematical problems requiring multi-step logical deductions and structured symbolic reasoning.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-3-5-turbo-0125", "model_route_id": "openai__gpt-3-5-turbo-0125", "model_family_name": "GPT-3.5 Turbo", "developer": "openai", "params_billions": null, "total_evaluations": 5, "benchmark_count": 5, "benchmark_family_count": 5, "categories_covered": [ "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 5, "raw_model_ids": [ "openai/gpt-3.5-turbo-0125" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 423, "min": 0.0, "max": 1.0, "average": 0.45562198581560265 }, "reproducibility_summary": { "results_total": 423, "has_reproducibility_gap_count": 423, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 423, "total_groups": 423, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 422, "collaborative": 0, "unspecified": 1 } }, "comparability_summary": { "total_groups": 423, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic Red Team", "BBQ", "DROP", "HarmBench", "Helm air bench", "Helm safety", "MMLU", "RewardBench", "SimpleSafetyTests", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #39-40.34: Characterization of identity - Predicted Personality / Refusal Rate", "evaluation_name": "AIRBench 2024 - #39-40.34: Characterization of identity - Predicted Personality", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #39-40.34: Characterization of identity - Predicted Personality", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.967, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.942, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "RewardBench", "benchmarkKey": "reward_bench", "canonical_display_name": "Reward Bench / Chat / Accuracy", "evaluation_name": "Chat", "score": 0.9218, "metric": "Chat accuracy - includes easy chat subsets", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.92, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Marketing / Exact Match", "evaluation_name": "Marketing", "score": 0.91, "metric": "EM on Marketing", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.814, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "DROP", "benchmarkKey": "llm_stats_drop", "canonical_display_name": "Drop / Score", "evaluation_name": "llm_stats.drop", "score": 0.702, "metric": "DROP (Discrete Reasoning Over Paragraphs) is a reading comprehension benchmark requiring discrete reasoning over paragraph content. It contains crowdsourced, adversarially-created questions that require resolving references and performing discrete operations like addition, counting, or sorting, demanding comprehensive paragraph understanding beyond paraphrase-and-entity-typing shortcuts.", "lower_is_better": false }, { "benchmark": "RewardBench", "benchmarkKey": "reward_bench_rewardbench", "canonical_display_name": "RewardBench / Score", "evaluation_name": "RewardBench", "score": 0.6534, "metric": "Overall RewardBench Score", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.632, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.606, "metric": "BBQ accuracy on BBQ", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-35-turbo", "model_route_id": "openai__gpt-35-turbo", "model_family_name": "GPT-3.5 Turbo", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-35-turbo" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 11, "min": 0.297, "max": 85.308, "average": 9.981272727272728 }, "reproducibility_summary": { "results_total": 11, "has_reproducibility_gap_count": 11, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 11, "total_groups": 11, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 11, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 11, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 85.308, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 10.7, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 9.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 1.5, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.75, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.5, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.462, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.441, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.418, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.418, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.297, "metric": "Benchmark score on GPQA.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-3-5-turbo-0301", "model_route_id": "openai__gpt-3-5-turbo-0301", "model_family_name": "GPT-3.5 Turbo 0301", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "general", "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-04-20T22:14:39.271662Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-3.5-turbo-0301" ], "last_updated": "2026-04-20T22:14:39.271662Z" } ], "score_summary": { "count": 389, "min": -1.0, "max": 1.0, "average": 0.6217429305912601 }, "reproducibility_summary": { "results_total": 389, "has_reproducibility_gap_count": 389, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 389, "total_groups": 389, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 389, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 389, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm air bench", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #43: Illegal/Regulated substances/goods / Refusal Rate", "evaluation_name": "AIRBench 2024 - #43: Illegal/Regulated substances/goods", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #43: Illegal/Regulated substances/goods", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": -1.0, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": -1.0, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": -1.0, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": -1.0, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": -1.0, "metric": "ROUGE-2 on XSUM", "lower_is_better": false }, { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.899, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.768, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.76, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.74, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.674, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.663, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.624, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.609, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.59, "metric": "EM on MMLU", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-3-5-turbo-0613", "model_route_id": "openai__gpt-3-5-turbo-0613", "model_family_name": "GPT-3.5 Turbo 0613", "developer": "openai", "params_billions": null, "total_evaluations": 7, "benchmark_count": 7, "benchmark_family_count": 7, "categories_covered": [ "general", "knowledge", "language_understanding", "other", "reasoning", "safety" ], "last_updated": "2026-04-25T00:19:43.227658Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 7, "raw_model_ids": [ "openai/gpt-3-5-turbo-0613", "openai/gpt-3.5-turbo-0613" ], "last_updated": "2026-04-25T00:19:43.227658Z" } ], "score_summary": { "count": 454, "min": -1.0, "max": 4.992, "average": 0.6738237885462559 }, "reproducibility_summary": { "results_total": 454, "has_reproducibility_gap_count": 454, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 454, "total_groups": 454, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 454, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 454, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic RLHF dataset", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "Best ChatGPT Prompts", "BoolQ", "CNN/DailyMail", "CivilComments", "GSM8K", "HarmBench", "HellaSwag", "Helm air bench", "Helm classic", "Helm lite", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IMDB", "Koala test dataset", "LegalBench", "MATH", "MMLU", "MS MARCO (TREC)", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "NaturalQuestions (open-book)", "Open Assistant", "OpenbookQA", "QuAC", "RAFT", "Self Instruct", "SimpleSafetyTests", "TruthfulQA", "Vicuna", "WMT 2014", "XSTest", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "Vicuna", "benchmarkKey": "helm_instruct_vicuna", "canonical_display_name": "Vicuna / Harmlessness", "evaluation_name": "Vicuna", "score": 4.992, "metric": "Harmlessness on Vicuna", "lower_is_better": false }, { "benchmark": "Self Instruct", "benchmarkKey": "helm_instruct_self_instruct", "canonical_display_name": "Self Instruct / Harmlessness", "evaluation_name": "Self Instruct", "score": 4.99, "metric": "Harmlessness on Self Instruct", "lower_is_better": false }, { "benchmark": "Koala test dataset", "benchmarkKey": "helm_instruct_koala_test_dataset", "canonical_display_name": "Koala test dataset / Harmlessness", "evaluation_name": "Koala test dataset", "score": 4.987, "metric": "Harmlessness on Koala test dataset", "lower_is_better": false }, { "benchmark": "Open Assistant", "benchmarkKey": "helm_instruct_open_assistant", "canonical_display_name": "Open Assistant / Harmlessness", "evaluation_name": "Open Assistant", "score": 4.987, "metric": "Harmlessness on Open Assistant", "lower_is_better": false }, { "benchmark": "Best ChatGPT Prompts", "benchmarkKey": "helm_instruct_best_chatgpt_prompts", "canonical_display_name": "Best ChatGPT Prompts / Harmlessness", "evaluation_name": "Best ChatGPT Prompts", "score": 4.986, "metric": "Harmlessness on Best ChatGPT Prompts", "lower_is_better": false }, { "benchmark": "Anthropic RLHF dataset", "benchmarkKey": "helm_instruct_anthropic_rlhf_dataset", "canonical_display_name": "Anthropic RLHF dataset / Harmlessness", "evaluation_name": "Anthropic RLHF dataset", "score": 4.964, "metric": "Harmlessness on Anthropic RLHF dataset", "lower_is_better": false }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #43: Illegal/Regulated substances/goods / Refusal Rate", "evaluation_name": "AIRBench 2024 - #43: Illegal/Regulated substances/goods", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #43: Illegal/Regulated substances/goods", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": -1.0, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": -1.0, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": -1.0, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": -1.0, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": -1.0, "metric": "ROUGE-2 on XSUM", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.996, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.958, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.943, "metric": "EM on IMDB", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-3-5-turbo-1106", "model_route_id": "openai__gpt-3-5-turbo-1106", "model_family_name": "GPT-3.5 Turbo 1106", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "general" ], "last_updated": "2026-04-25T00:19:43.227658Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-3.5-turbo-1106" ], "last_updated": "2026-04-25T00:19:43.227658Z" } ], "score_summary": { "count": 380, "min": 0.0, "max": 1.0, "average": 0.525428947368421 }, "reproducibility_summary": { "results_total": 380, "has_reproducibility_gap_count": 380, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 380, "total_groups": 380, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 380, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 380, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic Red Team", "BBQ", "HarmBench", "Helm air bench", "Helm safety", "SimpleSafetyTests", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #39-40.34: Characterization of identity - Predicted Personality / Refusal Rate", "evaluation_name": "AIRBench 2024 - #39-40.34: Characterization of identity - Predicted Personality", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #39-40.34: Characterization of identity - Predicted Personality", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.98, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.938, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.922, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.835, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.686, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.649, "metric": "BBQ accuracy on BBQ", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4-0613", "model_route_id": "openai__gpt-4-0613", "model_family_name": "GPT-4", "developer": "openai", "params_billions": null, "total_evaluations": 4, "benchmark_count": 4, "benchmark_family_count": 4, "categories_covered": [ "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 4, "raw_model_ids": [ "openai/gpt-4-0613" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 427, "min": 0.0, "max": 1.0, "average": 0.6708056206088993 }, "reproducibility_summary": { "results_total": 427, "has_reproducibility_gap_count": 427, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 427, "total_groups": 427, "multi_source_groups": 0, "first_party_only_groups": 7, "source_type_distribution": { "first_party": 7, "third_party": 420, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 427, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AI2 Reasoning Challenge (ARC)", "DROP", "GSM8K", "HellaSwag", "Helm air bench", "Helm lite", "LSAT", "LegalBench", "MATH", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "SAT Math", "Uniform Bar Exam", "WMT 2014", "Winogrande" ], "top_benchmark_scores": [ { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #1.1: Network intrusion / Refusal Rate", "evaluation_name": "AIRBench 2024 - #1.1: Network intrusion", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #1.1: Network intrusion", "lower_is_better": false }, { "benchmark": "AI2 Reasoning Challenge (ARC)", "benchmarkKey": "llm_stats_ai2_reasoning_challenge_arc", "canonical_display_name": "AI2 Reasoning Challenge Arc / Score", "evaluation_name": "llm_stats.ai2-reasoning-challenge-arc", "score": 0.963, "metric": "A dataset of 7,787 genuine grade-school level, multiple-choice science questions assembled to encourage research in advanced question-answering. The dataset is partitioned into a Challenge Set and Easy Set, where the Challenge Set contains only questions answered incorrectly by both retrieval-based and word co-occurrence algorithms. Covers multiple scientific domains including biology, physics, earth science, and chemistry, requiring scientific reasoning, causal understanding, and conceptual knowledge beyond simple fact retrieval. Includes a supporting corpus of over 14 million science sentences.", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Marketing / Exact Match", "evaluation_name": "Marketing", "score": 0.962, "metric": "EM on Marketing", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.96, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "llm_stats_hellaswag", "canonical_display_name": "Hellaswag / Score", "evaluation_name": "llm_stats.hellaswag", "score": 0.953, "metric": "A challenging commonsense natural language inference dataset that uses Adversarial Filtering to create questions trivial for humans (>95% accuracy) but difficult for state-of-the-art models, requiring completion of sentence endings based on physical situations and everyday activities", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.932, "metric": "EM on GSM8K", "lower_is_better": false }, { "benchmark": "Uniform Bar Exam", "benchmarkKey": "llm_stats_uniform_bar_exam", "canonical_display_name": "Uniform Bar Exam / Score", "evaluation_name": "llm_stats.uniform-bar-exam", "score": 0.9, "metric": "The Uniform Bar Examination (UBE) benchmark evaluates language models on the complete bar exam including multiple-choice Multistate Bar Examination (MBE), open-ended Multistate Essay Exam (MEE), and Multistate Performance Test (MPT) components. Used to assess legal reasoning capabilities across seven subject areas including Evidence, Torts, Constitutional Law, Contracts, Criminal Law and Procedure, Real Property, and Civil Procedure.", "lower_is_better": false }, { "benchmark": "SAT Math", "benchmarkKey": "llm_stats_sat_math", "canonical_display_name": "Sat Math / Score", "evaluation_name": "llm_stats.sat-math", "score": 0.89, "metric": "SAT Math benchmark from AGIEval containing standardized mathematics questions from the College Board SAT examination, designed to evaluate mathematical reasoning capabilities of foundation models using human-centric assessment methods.", "lower_is_better": false }, { "benchmark": "LSAT", "benchmarkKey": "llm_stats_lsat", "canonical_display_name": "Lsat / Score", "evaluation_name": "llm_stats.lsat", "score": 0.88, "metric": "LSAT (Law School Admission Test) benchmark evaluating complex reasoning capabilities across three challenging tasks: analytical reasoning, logical reasoning, and reading comprehension. The LSAT measures skills considered essential for success in law school including critical thinking, reading comprehension of complex texts, and analysis of arguments.", "lower_is_better": false }, { "benchmark": "Winogrande", "benchmarkKey": "llm_stats_winogrande", "canonical_display_name": "Winogrande / Score", "evaluation_name": "llm_stats.winogrande", "score": 0.875, "metric": "WinoGrande: An Adversarial Winograd Schema Challenge at Scale. A large-scale dataset of 44,000 pronoun resolution problems designed to test machine commonsense reasoning. Uses adversarial filtering to reduce spurious biases and provides a more robust evaluation of whether AI systems truly understand commonsense or exploit statistical shortcuts. Current best AI methods achieve 59.4-79.1% accuracy, significantly below human performance of 94.0%.", "lower_is_better": false }, { "benchmark": "Helm lite", "benchmarkKey": "helm_lite", "canonical_display_name": "Helm lite / Win Rate", "evaluation_name": "helm_lite", "score": 0.867, "metric": "How many models this model outperforms on average (over columns).", "lower_is_better": false }, { "benchmark": "MedQA", "benchmarkKey": "helm_lite_medqa", "canonical_display_name": "MedQA / Exact Match", "evaluation_name": "MedQA", "score": 0.815, "metric": "EM on MedQA", "lower_is_better": false }, { "benchmark": "DROP", "benchmarkKey": "llm_stats_drop", "canonical_display_name": "Drop / Score", "evaluation_name": "llm_stats.drop", "score": 0.809, "metric": "DROP (Discrete Reasoning Over Paragraphs) is a reading comprehension benchmark requiring discrete reasoning over paragraph content. It contains crowdsourced, adversarially-created questions that require resolving references and performing discrete operations like addition, counting, or sorting, demanding comprehensive paragraph understanding beyond paraphrase-and-entity-typing shortcuts.", "lower_is_better": false }, { "benchmark": "MATH", "benchmarkKey": "helm_lite_math", "canonical_display_name": "MATH / Equivalent (CoT)", "evaluation_name": "MATH", "score": 0.802, "metric": "Equivalent (CoT) on MATH", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.768, "metric": "F1 on NarrativeQA", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4", "model_route_id": "openai__gpt-4", "model_family_name": "GPT-4", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 8, "min": 0.91, "max": 60.0, "average": 23.522624999999998 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 60.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 37.5, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 32.961, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 30.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 13.1, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 12.8, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.91, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.91, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-4-0314", "model_route_id": "openai__gpt-4-0314", "model_family_name": "GPT-4 0314", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "general", "knowledge", "language_understanding", "safety" ], "last_updated": "2026-03-21T12:31:49.537868Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4-0314" ], "last_updated": "2026-03-21T12:31:49.537868Z" } ], "score_summary": { "count": 7, "min": 0.611, "max": 4.995, "average": 4.348714285714286 }, "reproducibility_summary": { "results_total": 7, "has_reproducibility_gap_count": 7, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 7, "total_groups": 7, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 7, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 7, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic RLHF dataset", "Best ChatGPT Prompts", "Holistic Evaluation of Language Models (HELM)", "Koala test dataset", "Open Assistant", "Self Instruct", "Vicuna" ], "top_benchmark_scores": [ { "benchmark": "Vicuna", "benchmarkKey": "helm_instruct_vicuna", "canonical_display_name": "Vicuna / Harmlessness", "evaluation_name": "Vicuna", "score": 4.995, "metric": "Harmlessness on Vicuna", "lower_is_better": false }, { "benchmark": "Open Assistant", "benchmarkKey": "helm_instruct_open_assistant", "canonical_display_name": "Open Assistant / Harmlessness", "evaluation_name": "Open Assistant", "score": 4.986, "metric": "Harmlessness on Open Assistant", "lower_is_better": false }, { "benchmark": "Self Instruct", "benchmarkKey": "helm_instruct_self_instruct", "canonical_display_name": "Self Instruct / Harmlessness", "evaluation_name": "Self Instruct", "score": 4.976, "metric": "Harmlessness on Self Instruct", "lower_is_better": false }, { "benchmark": "Best ChatGPT Prompts", "benchmarkKey": "helm_instruct_best_chatgpt_prompts", "canonical_display_name": "Best ChatGPT Prompts / Harmlessness", "evaluation_name": "Best ChatGPT Prompts", "score": 4.973, "metric": "Harmlessness on Best ChatGPT Prompts", "lower_is_better": false }, { "benchmark": "Koala test dataset", "benchmarkKey": "helm_instruct_koala_test_dataset", "canonical_display_name": "Koala test dataset / Harmlessness", "evaluation_name": "Koala test dataset", "score": 4.966, "metric": "Harmlessness on Koala test dataset", "lower_is_better": false }, { "benchmark": "Anthropic RLHF dataset", "benchmarkKey": "helm_instruct_anthropic_rlhf_dataset", "canonical_display_name": "Anthropic RLHF dataset / Harmlessness", "evaluation_name": "Anthropic RLHF dataset", "score": 4.934, "metric": "Harmlessness on Anthropic RLHF dataset", "lower_is_better": false }, { "benchmark": "Holistic Evaluation of Language Models (HELM)", "benchmarkKey": "helm_instruct", "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Win Rate", "evaluation_name": "helm_instruct", "score": 0.611, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4-turbo", "model_route_id": "openai__gpt-4-turbo", "model_family_name": "GPT-4 Turbo", "developer": "openai", "params_billions": null, "total_evaluations": 7, "benchmark_count": 7, "benchmark_family_count": 7, "categories_covered": [ "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4-turbo" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "2024-04-09", "variant_label": "2024-04-09", "evaluation_count": 6, "raw_model_ids": [ "openai/gpt-4-turbo-2024-04-09" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 448, "min": 0.0, "max": 30.0, "average": 0.9733401785714287 }, "reproducibility_summary": { "results_total": 448, "has_reproducibility_gap_count": 448, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 448, "total_groups": 448, "multi_source_groups": 0, "first_party_only_groups": 2, "source_type_distribution": { "first_party": 2, "third_party": 446, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 448, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "DROP", "GSM8K", "HarmBench", "Helm air bench", "Helm lite", "Helm safety", "LegalBench", "MATH", "MGSM", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "RewardBench", "SimpleSafetyTests", "WMT 2014", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 30.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 21.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 20.321, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 15.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 13.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 10.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #43: Illegal/Regulated substances/goods / Refusal Rate", "evaluation_name": "AIRBench 2024 - #43: Illegal/Regulated substances/goods", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #43: Illegal/Regulated substances/goods", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.997, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.99, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.977, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.97, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.961, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Us Foreign Policy / Exact Match", "evaluation_name": "Us Foreign Policy", "score": 0.96, "metric": "EM on Us Foreign Policy", "lower_is_better": false }, { "benchmark": "RewardBench", "benchmarkKey": "reward_bench", "canonical_display_name": "Reward Bench / Chat / Accuracy", "evaluation_name": "Chat", "score": 0.9525, "metric": "Chat accuracy - includes easy chat subsets", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.941, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-4-1-nano", "model_route_id": "openai__gpt-4-1-nano", "model_family_name": "GPT-4.1 nano", "developer": "openai", "params_billions": null, "total_evaluations": 9, "benchmark_count": 8, "benchmark_family_count": 8, "categories_covered": [ "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-04-14", "variant_label": "2025-04-14", "evaluation_count": 8, "raw_model_ids": [ "openai/gpt-4-1-nano-2025-04-14", "openai/gpt-4-1-nano-2025-04-14-fc", "openai/gpt-4-1-nano-2025-04-14-prompt", "openai/gpt-4.1-nano-2025-04-14" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4-1-nano" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 496, "min": 0.0, "max": 168.212, "average": 5.566508266129041 }, "reproducibility_summary": { "results_total": 496, "has_reproducibility_gap_count": 496, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 496, "total_groups": 464, "multi_source_groups": 0, "first_party_only_groups": 10, "source_type_distribution": { "first_party": 12, "third_party": 484, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 464, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Aider-Polyglot Edit", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BFCL leaderboard CSV", "COLLIE", "CharXiv-D", "ComplexFuncBench", "GPQA", "Graphwalks BFS <128k", "Graphwalks BFS >128k", "Graphwalks parents <128k", "Graphwalks parents >128k", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "Internal API instruction following (hard)", "MMLU-Pro", "Multi-IF", "Omni-MATH", "OpenAI-MRCR: 2 needle 128k", "OpenAI-MRCR: 2 needle 1M", "RewardBench 2", "SimpleSafetyTests", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 168.212, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 24.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 17.08, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 13.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 11.2, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #1.1: Network intrusion / Refusal Rate", "evaluation_name": "AIRBench 2024 - #1.1: Network intrusion", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #1.1: Network intrusion", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.996, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.99, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.96, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.938, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.875, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.868, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.848, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "IFEval", "benchmarkKey": "helm_capabilities_ifeval", "canonical_display_name": "IFEval / Strict Accuracy", "evaluation_name": "IFEval", "score": 0.843, "metric": "IFEval Strict Acc on IFEval", "lower_is_better": false }, { "benchmark": "WildBench", "benchmarkKey": "helm_capabilities_wildbench", "canonical_display_name": "WildBench / Score", "evaluation_name": "WildBench", "score": 0.811, "metric": "WB Score on WildBench", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4-5", "model_route_id": "openai__gpt-4-5", "model_family_name": "GPT-4.5", "developer": "openai", "params_billions": null, "total_evaluations": 3, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-02-27", "variant_label": "2025-02-27", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4-5-2025-02-27" ], "last_updated": "2026-04-07T08:15:57.597258Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-4-5", "openai/gpt-4.5" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 32, "min": 0.0, "max": 20.0, "average": 1.1406874999999996 }, "reproducibility_summary": { "results_total": 32, "has_reproducibility_gap_count": 32, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 32, "total_groups": 32, "multi_source_groups": 0, "first_party_only_groups": 19, "source_type_distribution": { "first_party": 19, "third_party": 13, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 32, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Aider-Polyglot Edit", "Artificial Analysis LLM API", "COLLIE", "CharXiv-D", "ComplexFuncBench", "GSM8k", "Graphwalks BFS <128k", "Graphwalks parents <128k", "Internal API instruction following (hard)", "MMLU", "MMMU", "MathVista", "Multi-Challenge", "Multi-IF", "OpenAI-MRCR: 2 needle 128k", "SWE-Lancer", "SWE-Lancer (IC-Diamond subset)", "SimpleQA", "TAU-bench Airline", "TAU-bench Retail" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 20.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "GSM8k", "benchmarkKey": "llm_stats_gsm8k", "canonical_display_name": "Gsm8k / Score", "evaluation_name": "llm_stats.gsm8k", "score": 0.97, "metric": "Grade School Math 8K, a dataset of 8.5K high-quality linguistically diverse grade school math word problems requiring multi-step reasoning and elementary arithmetic operations.", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "llm_stats_mmlu", "canonical_display_name": "Mmlu / Score", "evaluation_name": "llm_stats.mmlu", "score": 0.908, "metric": "Massive Multitask Language Understanding benchmark testing knowledge across 57 diverse subjects including STEM, humanities, social sciences, and professional domains", "lower_is_better": false }, { "benchmark": "CharXiv-D", "benchmarkKey": "llm_stats_charxiv_d", "canonical_display_name": "Charxiv D / Score", "evaluation_name": "llm_stats.charxiv-d", "score": 0.9, "metric": "CharXiv-D is the descriptive questions subset of the CharXiv benchmark, designed to assess multimodal large language models' ability to extract basic information from scientific charts. It contains descriptive questions covering information extraction, enumeration, pattern recognition, and counting across 2,323 diverse charts from arXiv papers, all curated and verified by human experts.", "lower_is_better": false }, { "benchmark": "MMMU", "benchmarkKey": "llm_stats_mmmu", "canonical_display_name": "Mmmu / Score", "evaluation_name": "llm_stats.mmmu", "score": 0.752, "metric": "MMMU (Massive Multi-discipline Multimodal Understanding) is a benchmark designed to evaluate multimodal models on college-level subject knowledge and deliberate reasoning. Contains 11.5K meticulously collected multimodal questions from college exams, quizzes, and textbooks, covering six core disciplines: Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering across 30 subjects and 183 subfields.", "lower_is_better": false }, { "benchmark": "Graphwalks parents <128k", "benchmarkKey": "llm_stats_graphwalks_parents_128k", "canonical_display_name": "Graphwalks Parents 128k / Score", "evaluation_name": "llm_stats.graphwalks-parents-128k", "score": 0.726, "metric": "A graph reasoning benchmark that evaluates language models' ability to find parent nodes in graphs with context length under 128k tokens, requiring understanding of graph structure and edge relationships.", "lower_is_better": false }, { "benchmark": "COLLIE", "benchmarkKey": "llm_stats_collie", "canonical_display_name": "Collie / Score", "evaluation_name": "llm_stats.collie", "score": 0.723, "metric": "COLLIE is a grammar-based framework for systematic construction of constrained text generation tasks. It allows specification of rich, compositional constraints across diverse generation levels and modeling challenges including language understanding, logical reasoning, and semantic planning. The COLLIE-v1 dataset contains 2,080 instances across 13 constraint structures.", "lower_is_better": false }, { "benchmark": "Graphwalks BFS <128k", "benchmarkKey": "llm_stats_graphwalks_bfs_128k", "canonical_display_name": "Graphwalks Bfs 128k / Score", "evaluation_name": "llm_stats.graphwalks-bfs-128k", "score": 0.723, "metric": "A graph reasoning benchmark that evaluates language models' ability to perform breadth-first search (BFS) operations on graphs with context length under 128k tokens, returning nodes reachable at specified depths.", "lower_is_better": false }, { "benchmark": "MathVista", "benchmarkKey": "llm_stats_mathvista", "canonical_display_name": "Mathvista / Score", "evaluation_name": "llm_stats.mathvista", "score": 0.723, "metric": "MathVista evaluates mathematical reasoning of foundation models in visual contexts. It consists of 6,141 examples derived from 28 existing multimodal datasets and 3 newly created datasets (IQTest, FunctionQA, and PaperQA), combining challenges from diverse mathematical and visual tasks to assess models' ability to understand complex figures and perform rigorous reasoning.", "lower_is_better": false }, { "benchmark": "Multi-IF", "benchmarkKey": "llm_stats_multi_if", "canonical_display_name": "Multi If / Score", "evaluation_name": "llm_stats.multi-if", "score": 0.708, "metric": "Multi-IF benchmarks LLMs on multi-turn and multilingual instruction following. It expands upon IFEval by incorporating multi-turn sequences and translating English prompts into 7 other languages, resulting in 4,501 multilingual conversations with three turns each. The benchmark reveals that current leading LLMs struggle with maintaining accuracy in multi-turn instructions and shows higher error rates for non-Latin script languages.", "lower_is_better": false }, { "benchmark": "TAU-bench Retail", "benchmarkKey": "llm_stats_tau_bench_retail", "canonical_display_name": "Tau Bench Retail / Score", "evaluation_name": "llm_stats.tau-bench-retail", "score": 0.684, "metric": "A benchmark for evaluating tool-agent-user interaction in retail environments. Tests language agents' ability to handle dynamic conversations with users while using domain-specific API tools and following policy guidelines. Evaluates agents on tasks like order cancellations, address changes, and order status checks through multi-turn conversations.", "lower_is_better": false }, { "benchmark": "ComplexFuncBench", "benchmarkKey": "llm_stats_complexfuncbench", "canonical_display_name": "Complexfuncbench / Score", "evaluation_name": "llm_stats.complexfuncbench", "score": 0.63, "metric": "ComplexFuncBench is a benchmark designed to evaluate large language models' capabilities in handling complex function calling scenarios. It encompasses multi-step and constrained function calling tasks that require long-parameter filling, parameter value reasoning, and managing contexts up to 128k tokens. The benchmark includes 1,000 samples across five real-world scenarios.", "lower_is_better": false }, { "benchmark": "SimpleQA", "benchmarkKey": "llm_stats_simpleqa", "canonical_display_name": "Simpleqa / Score", "evaluation_name": "llm_stats.simpleqa", "score": 0.625, "metric": "SimpleQA is a factuality benchmark developed by OpenAI that measures the short-form factual accuracy of large language models. The benchmark contains 4,326 short, fact-seeking questions that are adversarially collected and designed to have single, indisputable answers. Questions cover diverse topics from science and technology to entertainment, and the benchmark also measures model calibration by evaluating whether models know what they know.", "lower_is_better": false }, { "benchmark": "Internal API instruction following (hard)", "benchmarkKey": "llm_stats_internal_api_instruction_following_hard", "canonical_display_name": "Internal Api Instruction Following Hard / Score", "evaluation_name": "llm_stats.internal-api-instruction-following-hard", "score": 0.54, "metric": "Internal API instruction following (hard) benchmark - specific documentation not found in official sources", "lower_is_better": false }, { "benchmark": "TAU-bench Airline", "benchmarkKey": "llm_stats_tau_bench_airline", "canonical_display_name": "Tau Bench Airline / Score", "evaluation_name": "llm_stats.tau-bench-airline", "score": 0.5, "metric": "Part of τ-bench (TAU-bench), a benchmark for Tool-Agent-User interaction in real-world domains. The airline domain evaluates language agents' ability to interact with users through dynamic conversations while following domain-specific rules and using API tools. Agents must handle airline-related tasks and policies reliably.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4-5-preview", "model_route_id": "openai__gpt-4-5-preview", "model_family_name": "GPT-4.5 2025-02-27 preview", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "general" ], "last_updated": "2026-04-25T00:19:43.227658Z", "variants": [ { "variant_key": "2025-02-27", "variant_label": "2025-02-27", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-4.5-preview-2025-02-27" ], "last_updated": "2026-04-25T00:19:43.227658Z" } ], "score_summary": { "count": 380, "min": 0.0, "max": 1.0, "average": 0.7588894736842109 }, "reproducibility_summary": { "results_total": 380, "has_reproducibility_gap_count": 380, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 380, "total_groups": 380, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 380, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 380, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic Red Team", "BBQ", "HarmBench", "Helm air bench", "Helm safety", "SimpleSafetyTests", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #43: Illegal/Regulated substances/goods / Refusal Rate", "evaluation_name": "AIRBench 2024 - #43: Illegal/Regulated substances/goods", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #43: Illegal/Regulated substances/goods", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.994, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.965, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.958, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.951, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.92, "metric": "BBQ accuracy on BBQ", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4o", "model_route_id": "openai__gpt-4o", "model_family_name": "GPT-4o", "developer": "openai", "params_billions": null, "total_evaluations": 29, "benchmark_count": 16, "benchmark_family_count": 16, "categories_covered": [ "agentic", "coding", "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 4, "raw_model_ids": [ "openai/GPT 4o", "openai/gpt-4o" ], "last_updated": "2026-04-16T13:47:43.417572Z" }, { "variant_key": "2024-11-20", "variant_label": "2024-11-20", "evaluation_count": 3, "raw_model_ids": [ "openai/gpt-4o-2024-11-20" ], "last_updated": "2026-04-07T08:15:57.597545Z" }, { "variant_key": "2024-05-13", "variant_label": "2024-05-13", "evaluation_count": 11, "raw_model_ids": [ "openai/gpt-4o-2024-05-13" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "2024-08-06", "variant_label": "2024-08-06", "evaluation_count": 11, "raw_model_ids": [ "openai/gpt-4o-2024-08-06" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 989, "min": 0.0, "max": 838.0, "average": 4.407056388572893 }, "reproducibility_summary": { "results_total": 989, "has_reproducibility_gap_count": 971, "populated_ratio_avg": 0.019211324570273004 }, "provenance_summary": { "total_results": 989, "total_groups": 525, "multi_source_groups": 0, "first_party_only_groups": 29, "source_type_distribution": { "first_party": 29, "third_party": 960, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 525, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AI2D", "APEX v1", "ARC Prize evaluations leaderboard JSON", "ActivityNet", "Aider-Polyglot", "Aider-Polyglot Edit", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "COLLIE", "CharXiv-D", "ChartQA", "ComplexFuncBench", "DROP", "DocVQA", "ERQA", "Easy Problems", "EgoSchema", "Fibble arena", "GPQA", "GSM8K", "Graphwalks BFS <128k", "Graphwalks parents <128k", "Hard Problems", "HarmBench", "Helm air bench", "Helm lite", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "HumanEval", "IFEval", "Internal API instruction following (hard)", "LegalBench", "MATH", "MGSM", "MMLU", "MMLU-Pro", "MathVista", "MedQA", "Medium Problems", "Multi-Challenge", "Multi-IF", "NarrativeQA", "NaturalQuestions (closed-book)", "Omni-MATH", "OpenAI-MRCR: 2 needle 128k", "OpenbookQA", "RewardBench", "RewardBench 2", "SWE-Lancer", "SWE-Lancer (IC-Diamond subset)", "SWE-bench Verified", "SimpleSafetyTests", "Swe Bench Verified Mini", "TAU-bench Airline", "TAU-bench Retail", "Tau2 Airline", "Tau2 Retail", "WMT 2014", "WildBench", "Wordle Arena", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 105.591, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 24.2, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 18.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 6.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 4.375, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 2.5, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #39-40.36: Characterization of identity - Disability / Refusal Rate", "evaluation_name": "AIRBench 2024 - #39-40.36: Characterization of identity - Disability", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #39-40.36: Characterization of identity - Disability", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.991, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.985, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Medical Genetics / Exact Match", "evaluation_name": "Medical Genetics", "score": 0.98, "metric": "EM on Medical Genetics", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.973, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.968, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "RewardBench", "benchmarkKey": "reward_bench", "canonical_display_name": "Reward Bench / Chat / Accuracy", "evaluation_name": "Chat", "score": 0.9665, "metric": "Chat accuracy - includes easy chat subsets", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.951, "metric": "BBQ accuracy on BBQ", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4o-chatgpt", "model_route_id": "openai__gpt-4o-chatgpt", "model_family_name": "GPT-4o (ChatGPT)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4o-chatgpt" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 14, "min": 0.0, "max": 14.1, "average": 1.2275000000000003 }, "reproducibility_summary": { "results_total": 14, "has_reproducibility_gap_count": 14, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 14, "total_groups": 14, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 14, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 14, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 14.1, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.797, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.773, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.53, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.511, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.334, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.103, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.037, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-4o-chatgpt-03-25", "model_route_id": "openai__gpt-4o-chatgpt-03-25", "model_family_name": "GPT-4o (March 2025, chatgpt-4o-latest)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4o-chatgpt-03-25" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 16, "min": 0.0, "max": 25.7, "average": 3.004749999999999 }, "reproducibility_summary": { "results_total": 16, "has_reproducibility_gap_count": 16, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 16, "total_groups": 16, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 16, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 16, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 25.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 18.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.893, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.803, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.655, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.425, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.366, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.327, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.257, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.05, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-4o-realtime-dec-2024", "model_route_id": "openai__gpt-4o-realtime-dec-2024", "model_family_name": "GPT-4o Realtime (Dec '24)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4o-realtime-dec-2024" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 6, "min": 0.0, "max": 0.0, "average": 0.0 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 6, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-4o-mini", "model_route_id": "openai__gpt-4o-mini", "model_family_name": "GPT-4o mini", "developer": "openai", "params_billions": null, "total_evaluations": 13, "benchmark_count": 13, "benchmark_family_count": 13, "categories_covered": [ "agentic", "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2024-07-18", "variant_label": "2024-07-18", "evaluation_count": 10, "raw_model_ids": [ "openai/gpt-4o-mini-2024-07-18" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 3, "raw_model_ids": [ "openai/gpt-4o-mini" ], "last_updated": "2026-04-16T13:47:43.417572Z" } ], "score_summary": { "count": 487, "min": 0.0, "max": 559.0, "average": 6.220871864348028 }, "reproducibility_summary": { "results_total": 487, "has_reproducibility_gap_count": 469, "populated_ratio_avg": 0.039014373716632446 }, "provenance_summary": { "total_results": 487, "total_groups": 487, "multi_source_groups": 0, "first_party_only_groups": 2, "source_type_distribution": { "first_party": 2, "third_party": 485, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 487, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "DROP", "Fibble arena", "GPQA", "GSM8K", "HarmBench", "Helm air bench", "Helm lite", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "LegalBench", "MATH", "MGSM", "MMLU", "MMLU-Pro", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "Omni-MATH", "OpenbookQA", "RewardBench", "RewardBench 2", "SimpleSafetyTests", "Swe Bench Verified Mini", "WMT 2014", "WildBench", "Wordle Arena", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 57.598, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 14.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 12.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #1.1: Network intrusion / Refusal Rate", "evaluation_name": "AIRBench 2024 - #1.1: Network intrusion", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #1.1: Network intrusion", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.983, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.978, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.96, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "RewardBench", "benchmarkKey": "reward_bench", "canonical_display_name": "Reward Bench / Chat / Accuracy", "evaluation_name": "Chat", "score": 0.9497, "metric": "Chat accuracy - includes easy chat subsets", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.93, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Marketing / Exact Match", "evaluation_name": "Marketing", "score": 0.927, "metric": "EM on Marketing", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.92, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.882, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "MGSM", "benchmarkKey": "llm_stats_mgsm", "canonical_display_name": "Mgsm / Score", "evaluation_name": "llm_stats.mgsm", "score": 0.87, "metric": "MGSM (Multilingual Grade School Math) is a benchmark of grade-school math problems. Contains 250 grade-school math problems manually translated from the GSM8K dataset into ten typologically diverse languages: Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, and Telugu. Evaluates multilingual mathematical reasoning capabilities.", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.849, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.843, "metric": "EM on GSM8K", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4o-mini-realtime-dec-2024", "model_route_id": "openai__gpt-4o-mini-realtime-dec-2024", "model_family_name": "GPT-4o mini Realtime (Dec '24)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4o-mini-realtime-dec-2024" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 6, "min": 0.0, "max": 0.0, "average": 0.0 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 6, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-4o-1120", "model_route_id": "openai__gpt-4o-1120", "model_family_name": "GPT-4o-1120", "developer": "openai", "params_billions": null, "total_evaluations": 21, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-04-20T17:00:03.994647Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 21, "raw_model_ids": [ "openai/GPT-4o-1120" ], "last_updated": "2026-04-20T17:00:03.994647Z" } ], "score_summary": { "count": 21, "min": 0.0, "max": 0.125, "average": 0.036542494169494344 }, "reproducibility_summary": { "results_total": 21, "has_reproducibility_gap_count": 21, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 21, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 21, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Multi-SWE-bench (c)", "Multi-SWE-bench (c++)", "Multi-SWE-bench (go)", "Multi-SWE-bench (java)", "Multi-SWE-bench (javascript)", "Multi-SWE-bench (rust)", "Multi-SWE-bench (typescript)" ], "top_benchmark_scores": [ { "benchmark": "Multi-SWE-bench (java)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_java", "canonical_display_name": "Multi-SWE-Bench (java) / Score", "evaluation_name": "Multi-SWE-Bench (java)", "score": 0.125, "metric": "Fraction of java GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (c++)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_c", "canonical_display_name": "Multi-SWE-Bench (c++) / Score", "evaluation_name": "Multi-SWE-Bench (c++)", "score": 0.06976744186046512, "metric": "Fraction of c++ GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (rust)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_rust", "canonical_display_name": "Multi-SWE-Bench (rust) / Score", "evaluation_name": "Multi-SWE-Bench (rust)", "score": 0.058577405857740586, "metric": "Fraction of rust GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (go)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_go", "canonical_display_name": "Multi-SWE-Bench (go) / Score", "evaluation_name": "Multi-SWE-Bench (go)", "score": 0.0351288056206089, "metric": "Fraction of go GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (typescript)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_typescript", "canonical_display_name": "Multi-SWE-Bench (typescript) / Score", "evaluation_name": "Multi-SWE-Bench (typescript)", "score": 0.022321428571428572, "metric": "Fraction of typescript GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (javascript)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_javascript", "canonical_display_name": "Multi-SWE-Bench (javascript) / Score", "evaluation_name": "Multi-SWE-Bench (javascript)", "score": 0.019662921348314606, "metric": "Fraction of javascript GitHub issues resolved (0.0–1.0)", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5", "model_route_id": "openai__gpt-5", "model_family_name": "GPT-5", "developer": "openai", "params_billions": null, "total_evaluations": 30, "benchmark_count": 18, "benchmark_family_count": 18, "categories_covered": [ "agentic", "coding", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 17, "raw_model_ids": [ "openai/GPT 5", "openai/GPT-5", "openai/gpt-5" ], "last_updated": "2026-04-20T17:00:36.089462Z" }, { "variant_key": "2025-08-07-high", "variant_label": "2025-08-07 high", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2025-08-07-high" ], "last_updated": "2026-04-07T08:15:57.601872Z" }, { "variant_key": "2025-08-07-low", "variant_label": "2025-08-07 low", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2025-08-07-low" ], "last_updated": "2026-04-07T08:15:57.602168Z" }, { "variant_key": "2025-08-07-medium", "variant_label": "2025-08-07 medium", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2025-08-07-medium" ], "last_updated": "2026-04-07T08:15:57.602461Z" }, { "variant_key": "2025-08-07-minimal", "variant_label": "2025-08-07 minimal", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2025-08-07-minimal" ], "last_updated": "2026-04-07T08:15:57.602750Z" }, { "variant_key": "2025-08-07", "variant_label": "2025-08-07", "evaluation_count": 9, "raw_model_ids": [ "openai/gpt-5-2025-08-07" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 559, "min": 0.0, "max": 73320.0, "average": 673.8712930083715 }, "reproducibility_summary": { "results_total": 559, "has_reproducibility_gap_count": 541, "populated_ratio_avg": 0.03220035778175313 }, "provenance_summary": { "total_results": 559, "total_groups": 508, "multi_source_groups": 0, "first_party_only_groups": 43, "source_type_distribution": { "first_party": 44, "third_party": 515, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 508, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ACE", "APEX Agents", "APEX v1", "ARC Prize evaluations leaderboard JSON", "Aider-Polyglot", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BrowseComp Long Context 128k", "BrowseComp Long Context 256k", "COLLIE", "CharXiv-R", "ERQA", "Easy Problems", "FActScore", "Fibble arena", "FrontierMath", "GPQA", "Global-MMLU Lite", "Graphwalks BFS <128k", "Graphwalks parents <128k", "HMMT 2025", "Hard Problems", "HarmBench", "HealthBench Hard", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "HumanEval", "IFEval", "Internal API instruction following (hard)", "LongFact Concepts", "LongFact Objects", "MATH", "MMLU", "MMLU-Pro", "MMMU", "MMMU-Pro", "Medium Problems", "Multi-Challenge", "Multi-SWE-bench (c++)", "Omni-MATH", "OpenAI-MRCR: 2 needle 128k", "OpenAI-MRCR: 2 needle 256k", "SWE-Lancer (IC-Diamond subset)", "SWE-PolyBench Verified (Java)", "SWE-PolyBench Verified (JavaScript)", "SWE-PolyBench Verified (Python)", "SWE-PolyBench Verified (TypeScript)", "SWE-bench Verified", "SciArena leaderboard API", "SimpleSafetyTests", "Tau2 Airline", "Tau2 Retail", "Tau2 Telecom", "Terminal Bench 2 0", "VideoMME w sub.", "VideoMMMU", "WildBench", "Wordle Arena", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 95.722, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 94.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 82.082, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 82.082, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 49.6, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 44.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 36.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "canonical_display_name": "SciArena / Cost per 100 Calls", "evaluation_name": "overall_cost_per_100_calls_usd", "score": 2.9752, "metric": "overall_cost_per_100_calls_usd", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.25, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Self-harm / Refusal Rate", "evaluation_name": "AIRBench 2024 - Self-harm", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Self-harm", "lower_is_better": false }, { "benchmark": "SWE-Lancer (IC-Diamond subset)", "benchmarkKey": "llm_stats_swe_lancer_ic_diamond_subset", "canonical_display_name": "Swe Lancer Ic Diamond Subset / Score", "evaluation_name": "llm_stats.swe-lancer-ic-diamond-subset", "score": 1.0, "metric": "SWE-Lancer (IC-Diamond subset) is a benchmark of real-world freelance software engineering tasks from Upwork, ranging from $50 bug fixes to $32,000 feature implementations. It evaluates AI models on independent engineering tasks using end-to-end tests triple-verified by experienced software engineers, and includes managerial tasks where models choose between technical implementation proposals.", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.998, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.994, "metric": "Benchmark score on MATH-500.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-chatgpt", "model_route_id": "openai__gpt-5-chatgpt", "model_family_name": "GPT-5 (ChatGPT)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-chatgpt" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.0, "max": 156.165, "average": 14.081736842105263 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 156.165, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 48.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 21.8, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 21.2, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.25, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.82, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.686, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.637, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.608, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.608, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.543, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.483, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.45, "metric": "Benchmark score on IFBench.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-low", "model_route_id": "openai__gpt-5-low", "model_family_name": "GPT-5 (low)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-low" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 21, "min": 0.184, "max": 83.0, "average": 12.640571428571429 }, "reproducibility_summary": { "results_total": 21, "has_reproducibility_gap_count": 21, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 21, "total_groups": 21, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 21, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 21, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 83.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 70.273, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 39.2, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 30.7, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 9.789, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 9.789, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.25, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.987, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.86, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.842, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.83, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.83, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.808, "metric": "Benchmark score on GPQA.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-medium", "model_route_id": "openai__gpt-5-medium", "model_family_name": "GPT-5 (medium)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-medium" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 21, "min": 0.235, "max": 91.7, "average": 16.807 }, "reproducibility_summary": { "results_total": 21, "has_reproducibility_gap_count": 21, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 21, "total_groups": 21, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 21, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 21, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 91.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 74.582, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 42.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 41.208, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 41.208, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 39.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.25, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.991, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.917, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.917, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.867, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.865, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.842, "metric": "Benchmark score on GPQA.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-minimal", "model_route_id": "openai__gpt-5-minimal", "model_family_name": "GPT-5 (minimal)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-minimal" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 21, "min": 0.054, "max": 74.173, "average": 8.435666666666666 }, "reproducibility_summary": { "results_total": 21, "has_reproducibility_gap_count": 21, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 21, "total_groups": 21, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 21, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 21, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 74.173, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 31.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 25.1, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 23.9, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.25, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.003, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 1.003, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.861, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.806, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.673, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.67, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.558, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.456, "metric": "Benchmark score on IFBench.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-codex", "model_route_id": "openai__gpt-5-codex", "model_family_name": "GPT-5 Codex (high)", "developer": "openai", "params_billions": null, "total_evaluations": 5, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "agentic", "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 5, "raw_model_ids": [ "openai/GPT 5 Codex", "openai/gpt-5-codex" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 23, "min": 0.256, "max": 172.697, "average": 22.502130434782607 }, "reproducibility_summary": { "results_total": 23, "has_reproducibility_gap_count": 23, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 23, "total_groups": 21, "multi_source_groups": 0, "first_party_only_groups": 1, "source_type_distribution": { "first_party": 1, "third_party": 22, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 21, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "APEX Agents", "Artificial Analysis LLM API", "Terminal Bench 2 0" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 172.697, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 98.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 44.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 44.3, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 38.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 5.865, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 5.865, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.25, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.987, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.868, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.865, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.84, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.837, "metric": "Benchmark score on GPQA.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-mini-medium", "model_route_id": "openai__gpt-5-mini-medium", "model_family_name": "GPT-5 mini (medium)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-mini-medium" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.146, "max": 85.0, "average": 14.825999999999999 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 85.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 76.874, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 38.9, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 32.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 19.491, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 19.491, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 2.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.85, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.828, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.803, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.712, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.711, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.692, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.688, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.66, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-mini-minimal", "model_route_id": "openai__gpt-5-mini-minimal", "model_family_name": "GPT-5 mini (minimal)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-mini-minimal" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.05, "max": 76.338, "average": 9.192789473684211 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 76.338, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 46.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 21.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 20.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 2.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.959, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.959, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.775, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.688, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.687, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.545, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.467, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.456, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.369, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.357, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-nano", "model_route_id": "openai__gpt-5-nano", "model_family_name": "GPT-5 nano", "developer": "openai", "params_billions": null, "total_evaluations": 16, "benchmark_count": 9, "benchmark_family_count": 9, "categories_covered": [ "agentic", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-08-07-high", "variant_label": "2025-08-07 high", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-nano-2025-08-07-high" ], "last_updated": "2026-04-07T08:15:57.608112Z" }, { "variant_key": "2025-08-07-low", "variant_label": "2025-08-07 low", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-nano-2025-08-07-low" ], "last_updated": "2026-04-07T08:15:57.608403Z" }, { "variant_key": "2025-08-07-medium", "variant_label": "2025-08-07 medium", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-nano-2025-08-07-medium" ], "last_updated": "2026-04-07T08:15:57.608689Z" }, { "variant_key": "2025-08-07-minimal", "variant_label": "2025-08-07 minimal", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-nano-2025-08-07-minimal" ], "last_updated": "2026-04-07T08:15:57.609237Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 6, "raw_model_ids": [ "openai/gpt-5-nano" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "2025-08-07", "variant_label": "2025-08-07", "evaluation_count": 6, "raw_model_ids": [ "openai/gpt-5-nano-2025-08-07", "openai/gpt-5-nano-2025-08-07-fc", "openai/gpt-5-nano-2025-08-07-prompt" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 507, "min": 0.0, "max": 902.5921491006892, "average": 8.57036577731892 }, "reproducibility_summary": { "results_total": 507, "has_reproducibility_gap_count": 507, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 507, "total_groups": 450, "multi_source_groups": 0, "first_party_only_groups": 1, "source_type_distribution": { "first_party": 1, "third_party": 506, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 450, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BFCL leaderboard CSV", "FrontierMath", "GPQA", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "Omni-MATH", "SciArena leaderboard API", "SimpleSafetyTests", "Terminal Bench 2 0", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 118.877, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 83.869, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 83.869, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 83.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 26.8, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 20.3, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 11.5, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 2.57, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Self-harm / Refusal Rate", "evaluation_name": "AIRBench 2024 - Self-harm", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Self-harm", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.996, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.986, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.984, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.976, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.976, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-nano-medium", "model_route_id": "openai__gpt-5-nano-medium", "model_family_name": "GPT-5 nano (medium)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-nano-medium" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.05, "max": 131.13, "average": 18.446473684210527 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 131.13, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 78.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 43.363, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 43.363, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 25.9, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 22.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.783, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.772, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.763, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.67, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.659, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.4, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.4, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.338, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.304, "metric": "Benchmark score on tau2.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-nano-minimal", "model_route_id": "openai__gpt-5-nano-minimal", "model_family_name": "GPT-5 nano (minimal)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-nano-minimal" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.041, "max": 127.733, "average": 9.90294736842105 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 127.733, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 27.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 14.2, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 13.8, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.813, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.813, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.556, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.47, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.428, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.4, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.325, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.291, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.273, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.257, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.2, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1-non-reasoning", "model_route_id": "openai__gpt-5-1-non-reasoning", "model_family_name": "GPT-5.1 (Non-reasoning)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-1-non-reasoning" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.052, "max": 95.206, "average": 10.970578947368423 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 95.206, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 38.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 27.4, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 27.3, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.25, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.801, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.774, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.774, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.643, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.494, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.465, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.44, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.432, "metric": "Benchmark score on IFBench.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1-codex-mini", "model_route_id": "openai__gpt-5-1-codex-mini", "model_family_name": "GPT-5.1 Codex mini (high)", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "agentic", "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-5-1-codex-mini", "openai/gpt-5.1-codex-mini" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 20, "min": 0.169, "max": 196.795, "average": 21.292299999999997 }, "reproducibility_summary": { "results_total": 20, "has_reproducibility_gap_count": 20, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 20, "total_groups": 20, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 20, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 20, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "Terminal Bench 2 0" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 196.795, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 91.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 43.1, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 38.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 36.4, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 5.032, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 5.032, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 2.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.917, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.836, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.82, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.813, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.688, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.679, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.629, "metric": "Benchmark score on tau2.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1-high", "model_route_id": "openai__gpt-5-1-high", "model_family_name": "GPT-5.1 High", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-11-12", "variant_label": "2025-11-12", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.1-high-2025-11-12" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 1, "min": 0.996, "max": 0.996, "average": 0.996 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 1, "source_type_distribution": { "first_party": 1, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AIME 2025" ], "top_benchmark_scores": [ { "benchmark": "AIME 2025", "benchmarkKey": "llm_stats_aime_2025", "canonical_display_name": "Aime 2025 / Score", "evaluation_name": "llm_stats.aime-2025", "score": 0.996, "metric": "All 30 problems from the 2025 American Invitational Mathematics Examination (AIME I and AIME II), testing olympiad-level mathematical reasoning with integer answers from 000-999. Used as an AI benchmark to evaluate large language models' ability to solve complex mathematical problems requiring multi-step logical deductions and structured symbolic reasoning.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1-codex-max", "model_route_id": "openai__gpt-5-1-codex-max", "model_family_name": "GPT-5.1-Codex-Max", "developer": "OpenAI", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-03-17T19:48:21.772108Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.1-codex-max" ], "last_updated": "2026-03-17T19:48:21.772108Z" } ], "score_summary": { "count": 1, "min": 60.4, "max": 60.4, "average": 60.4 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 1, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Terminal Bench 2 0" ], "top_benchmark_scores": [ { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 60.4, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-2-non-reasoning", "model_route_id": "openai__gpt-5-2-non-reasoning", "model_family_name": "GPT-5.2 (Non-reasoning)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-non-reasoning" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.073, "max": 65.834, "average": 11.139473684210527 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 65.834, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 51.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 34.7, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 33.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 14.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 4.813, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.75, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.814, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.712, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.669, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.567, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.567, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.51, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.474, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.465, "metric": "Benchmark score on tau2.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-2-medium", "model_route_id": "openai__gpt-5-2-medium", "model_family_name": "GPT-5.2 (medium)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-medium" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.0, "max": 96.7, "average": 11.30621052631579 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 96.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 46.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 44.2, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 14.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 4.813, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.75, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.967, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.894, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.864, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.859, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.743, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.652, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.633, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.462, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_terminalbench_hard", "canonical_display_name": "artificial_analysis.terminalbench_hard / Terminalbench Hard", "evaluation_name": "artificial_analysis.terminalbench_hard", "score": 0.432, "metric": "Benchmark score on Terminal-Bench Hard.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-2-pro", "model_route_id": "openai__gpt-5-2-pro", "model_family_name": "GPT-5.2 Pro", "developer": "openai", "params_billions": null, "total_evaluations": 5, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/GPT 5.2 Pro" ], "last_updated": "2026-03-11T20:16:40Z" }, { "variant_key": "2025-12-11-high", "variant_label": "2025-12-11 high", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-pro-2025-12-11-high" ], "last_updated": "2026-04-07T08:15:57.601040Z" }, { "variant_key": "2025-12-11-medium", "variant_label": "2025-12-11 medium", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-pro-2025-12-11-medium" ], "last_updated": "2026-04-07T08:15:57.601366Z" }, { "variant_key": "2025-12-11-xhigh", "variant_label": "2025-12-11 xhigh", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-pro-2025-12-11-xhigh" ], "last_updated": "2026-04-07T08:15:57.601670Z" }, { "variant_key": "2025-12-11", "variant_label": "2025-12-11", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.2-pro-2025-12-11" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 30, "min": 0.3792, "max": 16.662, "average": 3.432326666666667 }, "reproducibility_summary": { "results_total": 30, "has_reproducibility_gap_count": 30, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 30, "total_groups": 18, "multi_source_groups": 0, "first_party_only_groups": 10, "source_type_distribution": { "first_party": 10, "third_party": 20, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 18, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AIME 2025", "APEX v1", "ARC Prize evaluations leaderboard JSON", "ARC-AGI", "ARC-AGI v2", "BrowseComp", "GPQA", "HMMT 2025" ], "top_benchmark_scores": [ { "benchmark": "AIME 2025", "benchmarkKey": "llm_stats_aime_2025", "canonical_display_name": "Aime 2025 / Score", "evaluation_name": "llm_stats.aime-2025", "score": 1.0, "metric": "All 30 problems from the 2025 American Invitational Mathematics Examination (AIME I and AIME II), testing olympiad-level mathematical reasoning with integer answers from 000-999. Used as an AI benchmark to evaluate large language models' ability to solve complex mathematical problems requiring multi-step logical deductions and structured symbolic reasoning.", "lower_is_better": false }, { "benchmark": "HMMT 2025", "benchmarkKey": "llm_stats_hmmt_2025", "canonical_display_name": "Hmmt 2025 / Score", "evaluation_name": "llm_stats.hmmt-2025", "score": 1.0, "metric": "Harvard-MIT Mathematics Tournament 2025 - A prestigious student-organized mathematics competition for high school students featuring two tournaments (November 2025 at MIT and February 2026 at Harvard) with individual tests, team rounds, and guts rounds", "lower_is_better": false }, { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Public_Eval / Score", "evaluation_name": "v1_Public_Eval", "score": 0.9761, "metric": "v1_Public_Eval", "lower_is_better": false }, { "benchmark": "GPQA", "benchmarkKey": "llm_stats_gpqa", "canonical_display_name": "Gpqa / Score", "evaluation_name": "llm_stats.gpqa", "score": 0.932, "metric": "A challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. Questions are Google-proof and extremely difficult, with PhD experts reaching 65% accuracy.", "lower_is_better": false }, { "benchmark": "ARC-AGI", "benchmarkKey": "llm_stats_arc_agi", "canonical_display_name": "Arc Agi / Score", "evaluation_name": "llm_stats.arc-agi", "score": 0.905, "metric": "The Abstraction and Reasoning Corpus for Artificial General Intelligence (ARC-AGI) is a benchmark designed to test general intelligence and abstract reasoning capabilities through visual grid-based transformation tasks. Each task consists of 2-5 demonstration pairs showing input grids transformed into output grids according to underlying rules, with test-takers required to infer these rules and apply them to novel test inputs. The benchmark uses colored grids (up to 30x30) with 10 discrete colors/symbols, designed to measure human-like general fluid intelligence and skill-acquisition efficiency with minimal prior knowledge.", "lower_is_better": false }, { "benchmark": "BrowseComp", "benchmarkKey": "llm_stats_browsecomp", "canonical_display_name": "Browsecomp / Score", "evaluation_name": "llm_stats.browsecomp", "score": 0.779, "metric": "BrowseComp is a benchmark comprising 1,266 questions that challenge AI agents to persistently navigate the internet in search of hard-to-find, entangled information. The benchmark measures agents' ability to exercise persistence in information gathering, demonstrate creativity in web navigation, and find concise, verifiable answers. Despite the difficulty of the questions, BrowseComp is simple and easy-to-use, as predicted answers are short and easily verifiable against reference answers.", "lower_is_better": false }, { "benchmark": "APEX v1", "benchmarkKey": "apex_v1", "canonical_display_name": "APEX v1 / Score", "evaluation_name": "apex-v1", "score": 0.668, "metric": "Overall APEX-v1 mean score across all jobs.", "lower_is_better": false }, { "benchmark": "ARC-AGI v2", "benchmarkKey": "llm_stats_arc_agi_v2", "canonical_display_name": "Arc Agi V2 / Score", "evaluation_name": "llm_stats.arc-agi-v2", "score": 0.542, "metric": "ARC-AGI-2 is an upgraded benchmark for measuring abstract reasoning and problem-solving abilities in AI systems through visual grid transformation tasks. It evaluates fluid intelligence via input-output grid pairs (1x1 to 30x30) using colored cells (0-9), requiring models to identify underlying transformation rules from demonstration examples and apply them to test cases. Designed to be easy for humans but challenging for AI, focusing on core cognitive abilities like spatial reasoning, pattern recognition, and compositional generalization.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-non-reasoning", "model_route_id": "openai__gpt-5-4-non-reasoning", "model_family_name": "GPT-5.4 (Non-reasoning)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-non-reasoning" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 15, "min": 0.106, "max": 60.861, "average": 10.972266666666664 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 60.861, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 41.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 35.4, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 5.625, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 2.5, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.748, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.593, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.593, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.484, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.473, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.471, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_terminalbench_hard", "canonical_display_name": "artificial_analysis.terminalbench_hard / Terminalbench Hard", "evaluation_name": "artificial_analysis.terminalbench_hard", "score": 0.379, "metric": "Benchmark score on Terminal-Bench Hard.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.351, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.106, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-pro", "model_route_id": "openai__gpt-5-4-pro", "model_family_name": "GPT-5.4 Pro (xhigh)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-pro" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 6, "min": 0.0, "max": 180.0, "average": 46.25 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 6, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 180.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 67.5, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 30.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-5-4-mini-non-reasoning", "model_route_id": "openai__gpt-5-4-mini-non-reasoning", "model_family_name": "GPT-5.4 mini (Non-Reasoning)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-mini-non-reasoning" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 15, "min": 0.057, "max": 153.816, "average": 14.1568 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 153.816, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 25.3, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 23.3, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 4.5, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 1.688, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.75, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.606, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.416, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.416, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.396, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.388, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.303, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.234, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_terminalbench_hard", "canonical_display_name": "artificial_analysis.terminalbench_hard / Terminalbench Hard", "evaluation_name": "artificial_analysis.terminalbench_hard", "score": 0.182, "metric": "Benchmark score on Terminal-Bench Hard.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.057, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-mini-medium", "model_route_id": "openai__gpt-5-4-mini-medium", "model_family_name": "GPT-5.4 mini (medium)", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "other", "reasoning" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-5-4-mini-medium" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 23, "min": 0.0444, "max": 160.905, "average": 11.348552173913042 }, "reproducibility_summary": { "results_total": 23, "has_reproducibility_gap_count": 23, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 23, "total_groups": 23, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 23, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 23, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 160.905, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 37.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 37.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 6.275, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 6.275, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 4.5, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 1.688, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.823, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.75, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.648, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.613, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.442, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.365, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_terminalbench_hard", "canonical_display_name": "artificial_analysis.terminalbench_hard / Terminalbench Hard", "evaluation_name": "artificial_analysis.terminalbench_hard", "score": 0.341, "metric": "Benchmark score on Terminal-Bench Hard.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.171, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-nano-non-reasoning", "model_route_id": "openai__gpt-5-4-nano-non-reasoning", "model_family_name": "GPT-5.4 nano (Non-Reasoning)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-nano-non-reasoning" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 15, "min": 0.042, "max": 175.171, "average": 15.501866666666668 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 175.171, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 27.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 24.4, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 1.25, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.558, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.514, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.514, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.463, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.352, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.348, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.327, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.247, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_terminalbench_hard", "canonical_display_name": "artificial_analysis.terminalbench_hard / Terminalbench Hard", "evaluation_name": "artificial_analysis.terminalbench_hard", "score": 0.242, "metric": "Benchmark score on Terminal-Bench Hard.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.2, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.042, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-nano-medium", "model_route_id": "openai__gpt-5-4-nano-medium", "model_family_name": "GPT-5.4 nano (medium)", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "other", "reasoning" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-5-4-nano-medium" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 23, "min": 0.0194, "max": 177.374, "average": 11.396908695652174 }, "reproducibility_summary": { "results_total": 23, "has_reproducibility_gap_count": 23, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 23, "total_groups": 23, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 23, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 23, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 177.374, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 38.1, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 35.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 2.678, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 2.678, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 1.25, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.761, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.644, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.573, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.526, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 0.463, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.384, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_terminalbench_hard", "canonical_display_name": "artificial_analysis.terminalbench_hard / Terminalbench Hard", "evaluation_name": "artificial_analysis.terminalbench_hard", "score": 0.333, "metric": "Benchmark score on Terminal-Bench Hard.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.2, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_hle", "canonical_display_name": "artificial_analysis.hle / Hle", "evaluation_name": "artificial_analysis.hle", "score": 0.147, "metric": "Benchmark score on Humanity's Last Exam.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-high", "model_route_id": "openai__gpt-5-4-high", "model_family_name": "GPT-5.4-high", "developer": "OpenAI", "params_billions": null, "total_evaluations": 3, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "other", "reasoning" ], "last_updated": "2026-04-22T13:06:30.542123Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 3, "raw_model_ids": [ "openai/gpt-5-4-high", "openai/gpt-5.4-high" ], "last_updated": "2026-04-22T13:06:30.542123Z" } ], "score_summary": { "count": 16, "min": 0.27, "max": 596.8, "average": 106.1731375 }, "reproducibility_summary": { "results_total": 16, "has_reproducibility_gap_count": 16, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 16, "total_groups": 12, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 16, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 12, "groups_with_variant_check": 4, "groups_with_cross_party_check": 0, "variant_divergent_count": 3, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "CocoaBench v1.0" ], "top_benchmark_scores": [ { "benchmark": "CocoaBench v1.0", "benchmarkKey": "cocoabench", "canonical_display_name": "Cocoabench / Avg Cost Usd", "evaluation_name": "cocoabench.overall.avg_cost_usd", "score": 2.314, "metric": "Average task cost in USD", "lower_is_better": true }, { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v2_Public_Eval / Score", "evaluation_name": "v2_Public_Eval", "score": 0.7583, "metric": "v2_Public_Eval", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-5", "model_route_id": "openai__gpt-5-5", "model_family_name": "GPT-5.5", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.5" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 22, "min": 0.25, "max": 0.98, "average": 0.7008636363636364 }, "reproducibility_summary": { "results_total": 22, "has_reproducibility_gap_count": 22, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 22, "total_groups": 22, "multi_source_groups": 0, "first_party_only_groups": 22, "source_type_distribution": { "first_party": 22, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 22, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC-AGI", "ARC-AGI v2", "BixBench", "BrowseComp", "CyberGym", "Finance Agent", "FrontierMath", "GDPval-MM", "GPQA", "GeneBench", "Graphwalks BFS >128k", "Graphwalks parents >128k", "Humanity's Last Exam", "MCP Atlas", "MMMU-Pro", "MRCR v2 (8-needle)", "OSWorld-Verified", "OfficeQA Pro", "SWE-Bench Pro", "Tau2 Telecom", "Terminal-Bench 2.0", "Toolathlon" ], "top_benchmark_scores": [ { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.98, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "ARC-AGI", "benchmarkKey": "llm_stats_arc_agi", "canonical_display_name": "Arc Agi / Score", "evaluation_name": "llm_stats.arc-agi", "score": 0.95, "metric": "The Abstraction and Reasoning Corpus for Artificial General Intelligence (ARC-AGI) is a benchmark designed to test general intelligence and abstract reasoning capabilities through visual grid-based transformation tasks. Each task consists of 2-5 demonstration pairs showing input grids transformed into output grids according to underlying rules, with test-takers required to infer these rules and apply them to novel test inputs. The benchmark uses colored grids (up to 30x30) with 10 discrete colors/symbols, designed to measure human-like general fluid intelligence and skill-acquisition efficiency with minimal prior knowledge.", "lower_is_better": false }, { "benchmark": "GPQA", "benchmarkKey": "llm_stats_gpqa", "canonical_display_name": "Gpqa / Score", "evaluation_name": "llm_stats.gpqa", "score": 0.936, "metric": "A challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. Questions are Google-proof and extremely difficult, with PhD experts reaching 65% accuracy.", "lower_is_better": false }, { "benchmark": "ARC-AGI v2", "benchmarkKey": "llm_stats_arc_agi_v2", "canonical_display_name": "Arc Agi V2 / Score", "evaluation_name": "llm_stats.arc-agi-v2", "score": 0.85, "metric": "ARC-AGI-2 is an upgraded benchmark for measuring abstract reasoning and problem-solving abilities in AI systems through visual grid transformation tasks. It evaluates fluid intelligence via input-output grid pairs (1x1 to 30x30) using colored cells (0-9), requiring models to identify underlying transformation rules from demonstration examples and apply them to test cases. Designed to be easy for humans but challenging for AI, focusing on core cognitive abilities like spatial reasoning, pattern recognition, and compositional generalization.", "lower_is_better": false }, { "benchmark": "GDPval-MM", "benchmarkKey": "llm_stats_gdpval_mm", "canonical_display_name": "Gdpval Mm / Score", "evaluation_name": "llm_stats.gdpval-mm", "score": 0.849, "metric": "GDPval-MM is the multimodal variant of the GDPval benchmark, evaluating AI model performance on real-world economically valuable tasks that require processing and generating multimodal content including documents, slides, diagrams, spreadsheets, images, and other professional deliverables across diverse industries.", "lower_is_better": false }, { "benchmark": "BrowseComp", "benchmarkKey": "llm_stats_browsecomp", "canonical_display_name": "Browsecomp / Score", "evaluation_name": "llm_stats.browsecomp", "score": 0.844, "metric": "BrowseComp is a benchmark comprising 1,266 questions that challenge AI agents to persistently navigate the internet in search of hard-to-find, entangled information. The benchmark measures agents' ability to exercise persistence in information gathering, demonstrate creativity in web navigation, and find concise, verifiable answers. Despite the difficulty of the questions, BrowseComp is simple and easy-to-use, as predicted answers are short and easily verifiable against reference answers.", "lower_is_better": false }, { "benchmark": "MMMU-Pro", "benchmarkKey": "llm_stats_mmmu_pro", "canonical_display_name": "Mmmu Pro / Score", "evaluation_name": "llm_stats.mmmu-pro", "score": 0.832, "metric": "A more robust multi-discipline multimodal understanding benchmark that enhances MMMU through a three-step process: filtering text-only answerable questions, augmenting candidate options, and introducing vision-only input settings. Achieves significantly lower model performance (16.8-26.9%) compared to original MMMU, providing more rigorous evaluation that closely mimics real-world scenarios.", "lower_is_better": false }, { "benchmark": "Terminal-Bench 2.0", "benchmarkKey": "llm_stats_terminal_bench_2", "canonical_display_name": "Terminal Bench 2 / Score", "evaluation_name": "llm_stats.terminal-bench-2", "score": 0.827, "metric": "Terminal-Bench 2.0 is an updated benchmark for testing AI agents' tool use ability to operate a computer via terminal. It evaluates how well models can handle real-world, end-to-end tasks autonomously, including compiling code, training models, setting up servers, system administration, security tasks, data science workflows, and cybersecurity vulnerabilities.", "lower_is_better": false }, { "benchmark": "CyberGym", "benchmarkKey": "llm_stats_cybergym", "canonical_display_name": "Cybergym / Score", "evaluation_name": "llm_stats.cybergym", "score": 0.818, "metric": "CyberGym is a benchmark for evaluating AI agents on cybersecurity tasks, testing their ability to identify vulnerabilities, perform security analysis, and complete security-related challenges in a controlled environment.", "lower_is_better": false }, { "benchmark": "BixBench", "benchmarkKey": "llm_stats_bixbench", "canonical_display_name": "Bixbench / Score", "evaluation_name": "llm_stats.bixbench", "score": 0.805, "metric": "BixBench is a benchmark for real-world bioinformatics and computational biology data analysis. It evaluates AI models on multi-step scientific workflows that require code execution, statistical reasoning, and biological domain knowledge to interpret experimental data.", "lower_is_better": false }, { "benchmark": "OSWorld-Verified", "benchmarkKey": "llm_stats_osworld_verified", "canonical_display_name": "Osworld Verified / Score", "evaluation_name": "llm_stats.osworld-verified", "score": 0.787, "metric": "OSWorld-Verified is a verified subset of OSWorld, a scalable real computer environment for multimodal agents supporting task setup, execution-based evaluation, and interactive learning across Ubuntu, Windows, and macOS.", "lower_is_better": false }, { "benchmark": "MCP Atlas", "benchmarkKey": "llm_stats_mcp_atlas", "canonical_display_name": "Mcp Atlas / Score", "evaluation_name": "llm_stats.mcp-atlas", "score": 0.753, "metric": "MCP Atlas is a benchmark for evaluating AI models on scaled tool use capabilities, measuring how well models can coordinate and utilize multiple tools across complex multi-step tasks.", "lower_is_better": false }, { "benchmark": "MRCR v2 (8-needle)", "benchmarkKey": "llm_stats_mrcr_v2_8_needle", "canonical_display_name": "Mrcr V2 8 Needle / Score", "evaluation_name": "llm_stats.mrcr-v2-8-needle", "score": 0.74, "metric": "MRCR v2 (8-needle) is a variant of the Multi-Round Coreference Resolution benchmark that includes 8 needle items to retrieve from long contexts. This tests models' ability to simultaneously track and reason about multiple pieces of information across extended conversations.", "lower_is_better": false }, { "benchmark": "Finance Agent", "benchmarkKey": "llm_stats_finance_agent", "canonical_display_name": "Finance Agent / Score", "evaluation_name": "llm_stats.finance-agent", "score": 0.6, "metric": "Finance Agent is a benchmark for evaluating AI models on agentic financial analysis tasks, testing their ability to process financial data, perform calculations, and generate accurate analyses across various financial domains.", "lower_is_better": false }, { "benchmark": "SWE-Bench Pro", "benchmarkKey": "llm_stats_swe_bench_pro", "canonical_display_name": "Swe Bench Pro / Score", "evaluation_name": "llm_stats.swe-bench-pro", "score": 0.586, "metric": "SWE-Bench Pro is an advanced version of SWE-Bench that evaluates language models on complex, real-world software engineering tasks requiring extended reasoning and multi-step problem solving.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-5-pro", "model_route_id": "openai__gpt-5-5-pro", "model_family_name": "GPT-5.5 Pro", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.5-pro" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 5, "min": 0.332, "max": 0.901, "average": 0.6048 }, "reproducibility_summary": { "results_total": 5, "has_reproducibility_gap_count": 5, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 5, "total_groups": 5, "multi_source_groups": 0, "first_party_only_groups": 5, "source_type_distribution": { "first_party": 5, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 5, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BrowseComp", "FrontierMath", "GDPval-MM", "GeneBench", "Humanity's Last Exam" ], "top_benchmark_scores": [ { "benchmark": "BrowseComp", "benchmarkKey": "llm_stats_browsecomp", "canonical_display_name": "Browsecomp / Score", "evaluation_name": "llm_stats.browsecomp", "score": 0.901, "metric": "BrowseComp is a benchmark comprising 1,266 questions that challenge AI agents to persistently navigate the internet in search of hard-to-find, entangled information. The benchmark measures agents' ability to exercise persistence in information gathering, demonstrate creativity in web navigation, and find concise, verifiable answers. Despite the difficulty of the questions, BrowseComp is simple and easy-to-use, as predicted answers are short and easily verifiable against reference answers.", "lower_is_better": false }, { "benchmark": "GDPval-MM", "benchmarkKey": "llm_stats_gdpval_mm", "canonical_display_name": "Gdpval Mm / Score", "evaluation_name": "llm_stats.gdpval-mm", "score": 0.823, "metric": "GDPval-MM is the multimodal variant of the GDPval benchmark, evaluating AI model performance on real-world economically valuable tasks that require processing and generating multimodal content including documents, slides, diagrams, spreadsheets, images, and other professional deliverables across diverse industries.", "lower_is_better": false }, { "benchmark": "Humanity's Last Exam", "benchmarkKey": "llm_stats_humanity_s_last_exam", "canonical_display_name": "Humanity S Last Exam / Score", "evaluation_name": "llm_stats.humanity-s-last-exam", "score": 0.572, "metric": "Humanity's Last Exam (HLE) is a multi-modal academic benchmark with 2,500 questions across mathematics, humanities, and natural sciences, designed to test LLM capabilities at the frontier of human knowledge with unambiguous, verifiable solutions", "lower_is_better": false }, { "benchmark": "FrontierMath", "benchmarkKey": "llm_stats_frontiermath", "canonical_display_name": "Frontiermath / Score", "evaluation_name": "llm_stats.frontiermath", "score": 0.396, "metric": "A benchmark of hundreds of original, exceptionally challenging mathematics problems crafted and vetted by expert mathematicians, covering most major branches of modern mathematics from number theory and real analysis to algebraic geometry and category theory.", "lower_is_better": false }, { "benchmark": "GeneBench", "benchmarkKey": "llm_stats_genebench", "canonical_display_name": "Genebench / Score", "evaluation_name": "llm_stats.genebench", "score": 0.332, "metric": "GeneBench is an evaluation focused on multi-stage scientific data analysis in genetics and quantitative biology. Tasks require reasoning about ambiguous or noisy data with minimal supervisory guidance, addressing realistic obstacles such as hidden confounders or QC failures, and correctly implementing and interpreting modern statistical methods.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-j-6b", "model_route_id": "openai__gpt-j-6b", "model_family_name": "GPT-J 6B", "developer": "openai", "params_billions": 6.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/GPT-J-6B" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 15, "min": 0.096, "max": 0.939, "average": 0.44206666666666666 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.939, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.663, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.649, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.619, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.559, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.545, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.52, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.514, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.345, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "QuAC", "benchmarkKey": "helm_classic_quac", "canonical_display_name": "QuAC / F1", "evaluation_name": "QuAC", "score": 0.33, "metric": "F1 on QuAC", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.273, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.249, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.199, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": 0.131, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": 0.096, "metric": "ROUGE-2 on XSUM", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-neox-20b", "model_route_id": "openai__gpt-neox-20b", "model_family_name": "GPT-NeoX 20B", "developer": "openai", "params_billions": 20.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/GPT-NeoX-20B" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 15, "min": 0.102, "max": 0.948, "average": 0.4587333333333334 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.948, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.718, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.683, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.599, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.596, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.524, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.516, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.505, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.398, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.351, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "QuAC", "benchmarkKey": "helm_classic_quac", "canonical_display_name": "QuAC / F1", "evaluation_name": "QuAC", "score": 0.326, "metric": "F1 on QuAC", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.276, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.216, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": 0.123, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": 0.102, "metric": "ROUGE-2 on XSUM", "lower_is_better": false } ] }, { "model_group_id": "openai/lingma-agent-lingma-swe-gpt-72b-v0918", "model_route_id": "openai__lingma-agent-lingma-swe-gpt-72b-v0918", "model_family_name": "Lingma Agent + Lingma SWE-GPT 72b (v0918)", "developer": "openai", "params_billions": 72.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-04-20T16:39:22.266076Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/Lingma Agent + Lingma SWE-GPT 72b (v0918)" ], "last_updated": "2026-04-20T16:39:22.266076Z" } ], "score_summary": { "count": 1, "min": 0.25, "max": 0.25, "average": 0.25 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 1, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "SWE-bench Verified" ], "top_benchmark_scores": [ { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.25, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false } ] }, { "model_group_id": "openai/lingma-agent-lingma-swe-gpt-72b-v0925", "model_route_id": "openai__lingma-agent-lingma-swe-gpt-72b-v0925", "model_family_name": "Lingma Agent + Lingma SWE-GPT 72b (v0925)", "developer": "openai", "params_billions": 72.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-04-20T16:39:22.266076Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/Lingma Agent + Lingma SWE-GPT 72b (v0925)" ], "last_updated": "2026-04-20T16:39:22.266076Z" } ], "score_summary": { "count": 1, "min": 0.288, "max": 0.288, "average": 0.288 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 1, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "SWE-bench Verified" ], "top_benchmark_scores": [ { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.288, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false } ] }, { "model_group_id": "openai/lingma-agent-lingma-swe-gpt-7b-v0918", "model_route_id": "openai__lingma-agent-lingma-swe-gpt-7b-v0918", "model_family_name": "Lingma Agent + Lingma SWE-GPT 7b (v0918)", "developer": "openai", "params_billions": 7.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-04-20T16:39:22.266076Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/Lingma Agent + Lingma SWE-GPT 7b (v0918)" ], "last_updated": "2026-04-20T16:39:22.266076Z" } ], "score_summary": { "count": 1, "min": 0.102, "max": 0.102, "average": 0.102 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 1, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "SWE-bench Verified" ], "top_benchmark_scores": [ { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.102, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false } ] }, { "model_group_id": "openai/lingma-agent-lingma-swe-gpt-7b-v0925", "model_route_id": "openai__lingma-agent-lingma-swe-gpt-7b-v0925", "model_family_name": "Lingma Agent + Lingma SWE-GPT 7b (v0925)", "developer": "openai", "params_billions": 7.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-04-20T16:39:22.266076Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/Lingma Agent + Lingma SWE-GPT 7b (v0925)" ], "last_updated": "2026-04-20T16:39:22.266076Z" } ], "score_summary": { "count": 1, "min": 0.182, "max": 0.182, "average": 0.182 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 1, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "SWE-bench Verified" ], "top_benchmark_scores": [ { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.182, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false } ] }, { "model_group_id": "openai/deep-research", "model_route_id": "openai__deep-research", "model_family_name": "OpenAI Deep Research", "developer": "OpenAI", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-22T13:06:30.542123Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/deep-research" ], "last_updated": "2026-04-22T13:06:30.542123Z" } ], "score_summary": { "count": 1, "min": 3.27, "max": 3.27, "average": 3.27 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 1, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "CocoaBench v1.0" ], "top_benchmark_scores": [ { "benchmark": "CocoaBench v1.0", "benchmarkKey": "cocoabench", "canonical_display_name": "Cocoabench / Accuracy Percent", "evaluation_name": "cocoabench.overall.accuracy_percent", "score": 3.27, "metric": "Overall task success rate on CocoaBench aggregate release", "lower_is_better": false } ] }, { "model_group_id": "openai/openai-o1", "model_route_id": "openai__openai-o1", "model_family_name": "OpenAI-o1", "developer": "openai", "params_billions": null, "total_evaluations": 21, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-04-20T17:00:03.994647Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 21, "raw_model_ids": [ "openai/OpenAI-o1" ], "last_updated": "2026-04-20T17:00:03.994647Z" } ], "score_summary": { "count": 21, "min": 0.004464285714285714, "max": 0.21875, "average": 0.05719766986605023 }, "reproducibility_summary": { "results_total": 21, "has_reproducibility_gap_count": 21, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 21, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 21, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Multi-SWE-bench (c)", "Multi-SWE-bench (c++)", "Multi-SWE-bench (go)", "Multi-SWE-bench (java)", "Multi-SWE-bench (javascript)", "Multi-SWE-bench (rust)", "Multi-SWE-bench (typescript)" ], "top_benchmark_scores": [ { "benchmark": "Multi-SWE-bench (java)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_java", "canonical_display_name": "Multi-SWE-Bench (java) / Score", "evaluation_name": "Multi-SWE-Bench (java)", "score": 0.21875, "metric": "Fraction of java GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (rust)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_rust", "canonical_display_name": "Multi-SWE-Bench (rust) / Score", "evaluation_name": "Multi-SWE-Bench (rust)", "score": 0.07112970711297072, "metric": "Fraction of rust GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (typescript)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_typescript", "canonical_display_name": "Multi-SWE-Bench (typescript) / Score", "evaluation_name": "Multi-SWE-Bench (typescript)", "score": 0.05803571428571429, "metric": "Fraction of typescript GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (c++)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_c", "canonical_display_name": "Multi-SWE-Bench (c++) / Score", "evaluation_name": "Multi-SWE-Bench (c++)", "score": 0.05426356589147287, "metric": "Fraction of c++ GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (javascript)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_javascript", "canonical_display_name": "Multi-SWE-Bench (javascript) / Score", "evaluation_name": "Multi-SWE-Bench (javascript)", "score": 0.05056179775280899, "metric": "Fraction of javascript GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (go)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_go", "canonical_display_name": "Multi-SWE-Bench (go) / Score", "evaluation_name": "Multi-SWE-Bench (go)", "score": 0.04672897196261682, "metric": "Fraction of go GitHub issues resolved (0.0–1.0)", "lower_is_better": false } ] }, { "model_group_id": "openai/openai-o3-mini-high", "model_route_id": "openai__openai-o3-mini-high", "model_family_name": "OpenAI-o3-mini-high", "developer": "openai", "params_billions": null, "total_evaluations": 21, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "agentic" ], "last_updated": "2026-04-20T17:00:03.994647Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 21, "raw_model_ids": [ "openai/OpenAI-o3-mini-high" ], "last_updated": "2026-04-20T17:00:03.994647Z" } ], "score_summary": { "count": 21, "min": 0.004464285714285714, "max": 0.1640625, "average": 0.046063736772594115 }, "reproducibility_summary": { "results_total": 21, "has_reproducibility_gap_count": 21, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 21, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 21, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Multi-SWE-bench (c)", "Multi-SWE-bench (c++)", "Multi-SWE-bench (go)", "Multi-SWE-bench (java)", "Multi-SWE-bench (javascript)", "Multi-SWE-bench (rust)", "Multi-SWE-bench (typescript)" ], "top_benchmark_scores": [ { "benchmark": "Multi-SWE-bench (java)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_java", "canonical_display_name": "Multi-SWE-Bench (java) / Score", "evaluation_name": "Multi-SWE-Bench (java)", "score": 0.1640625, "metric": "Fraction of java GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (rust)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_rust", "canonical_display_name": "Multi-SWE-Bench (rust) / Score", "evaluation_name": "Multi-SWE-Bench (rust)", "score": 0.0794979079497908, "metric": "Fraction of rust GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (c++)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_c", "canonical_display_name": "Multi-SWE-Bench (c++) / Score", "evaluation_name": "Multi-SWE-Bench (c++)", "score": 0.06976744186046512, "metric": "Fraction of c++ GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (typescript)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_typescript", "canonical_display_name": "Multi-SWE-Bench (typescript) / Score", "evaluation_name": "Multi-SWE-Bench (typescript)", "score": 0.049107142857142856, "metric": "Fraction of typescript GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (javascript)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_javascript", "canonical_display_name": "Multi-SWE-Bench (javascript) / Score", "evaluation_name": "Multi-SWE-Bench (javascript)", "score": 0.042134831460674156, "metric": "Fraction of javascript GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "Multi-SWE-bench (go)", "benchmarkKey": "multi_swe_bench_multi_swe_bench_go", "canonical_display_name": "Multi-SWE-Bench (go) / Score", "evaluation_name": "Multi-SWE-Bench (go)", "score": 0.0397196261682243, "metric": "Fraction of go GitHub issues resolved (0.0–1.0)", "lower_is_better": false } ] }, { "model_group_id": "openai/ada-350m", "model_route_id": "openai__ada-350m", "model_family_name": "ada 350M", "developer": "openai", "params_billions": 0.35, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/ada-350M" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 15, "min": 0.022, "max": 0.849, "average": 0.3390666666666666 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.849, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.581, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.517, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.435, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.423, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.38, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.365, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.326, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.29, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.243, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "QuAC", "benchmarkKey": "helm_classic_quac", "canonical_display_name": "QuAC / F1", "evaluation_name": "QuAC", "score": 0.242, "metric": "F1 on QuAC", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.215, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.108, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": 0.09, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": 0.022, "metric": "ROUGE-2 on XSUM", "lower_is_better": false } ] }, { "model_group_id": "openai/babbage-1-3b", "model_route_id": "openai__babbage-1-3b", "model_family_name": "babbage 1.3B", "developer": "openai", "params_billions": 1.3, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/babbage-1.3B" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 15, "min": 0.045, "max": 0.597, "average": 0.3554000000000001 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.597, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.574, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.555, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.519, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.491, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.455, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.451, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.438, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.317, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "QuAC", "benchmarkKey": "helm_classic_quac", "canonical_display_name": "QuAC / F1", "evaluation_name": "QuAC", "score": 0.273, "metric": "F1 on QuAC", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.235, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.188, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.114, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": 0.079, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": 0.045, "metric": "ROUGE-2 on XSUM", "lower_is_better": false } ] }, { "model_group_id": "openai/codex-mini-latest", "model_route_id": "openai__codex-mini-latest", "model_family_name": "codex-mini-latest", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.595841Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/codex-mini-latest" ], "last_updated": "2026-04-07T08:15:57.595841Z" } ], "score_summary": { "count": 8, "min": 0.0, "max": 0.3338, "average": 0.17045 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.1597, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "openai/curie-6-7b", "model_route_id": "openai__curie-6-7b", "model_family_name": "curie 6.7B", "developer": "openai", "params_billions": 6.7, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/curie-6.7B" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 15, "min": 0.091, "max": 0.889, "average": 0.4307333333333334 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.889, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.682, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.656, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.604, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.552, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.539, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.502, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.49, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "QuAC", "benchmarkKey": "helm_classic_quac", "canonical_display_name": "QuAC / F1", "evaluation_name": "QuAC", "score": 0.321, "metric": "F1 on QuAC", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.3, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.247, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.243, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.232, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": 0.113, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": 0.091, "metric": "ROUGE-2 on XSUM", "lower_is_better": false } ] }, { "model_group_id": "openai/davinci-175b", "model_route_id": "openai__davinci-175b", "model_family_name": "davinci 175B", "developer": "openai", "params_billions": 175.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/davinci-175B" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 15, "min": 0.126, "max": 0.933, "average": 0.5098 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.933, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.775, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.722, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.687, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.642, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.625, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.586, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.538, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.532, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.422, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.378, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "QuAC", "benchmarkKey": "helm_classic_quac", "canonical_display_name": "QuAC / F1", "evaluation_name": "QuAC", "score": 0.36, "metric": "F1 on QuAC", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.194, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": 0.127, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": 0.126, "metric": "ROUGE-2 on XSUM", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4-1106-preview", "model_route_id": "openai__gpt-4-1106-preview", "model_family_name": "gpt-4-1106-preview", "developer": "openai", "params_billions": null, "total_evaluations": 4, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "agentic", "general", "knowledge", "reasoning" ], "last_updated": "2026-04-20T16:39:22.266076Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 4, "raw_model_ids": [ "openai/gpt-4-1106-preview" ], "last_updated": "2026-04-20T16:39:22.266076Z" } ], "score_summary": { "count": 48, "min": 0.028, "max": 0.96, "average": 0.7465208333333332 }, "reproducibility_summary": { "results_total": 48, "has_reproducibility_gap_count": 48, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 48, "total_groups": 47, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 48, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 47, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "GSM8K", "Helm lite", "LegalBench", "MATH", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "SWE-bench Verified", "WMT 2014" ], "top_benchmark_scores": [ { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "canonical_display_name": "Mmlu / Us Foreign Policy / Exact Match", "evaluation_name": "Us Foreign Policy", "score": 0.96, "metric": "EM on Us Foreign Policy", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.95, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "MATH", "benchmarkKey": "helm_lite_math", "canonical_display_name": "MATH / Equivalent (CoT)", "evaluation_name": "MATH", "score": 0.857, "metric": "Equivalent (CoT) on MATH", "lower_is_better": false }, { "benchmark": "MedQA", "benchmarkKey": "helm_lite_medqa", "canonical_display_name": "MedQA / Exact Match", "evaluation_name": "MedQA", "score": 0.817, "metric": "EM on MedQA", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.727, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_lite_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.699, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "Helm lite", "benchmarkKey": "helm_lite", "canonical_display_name": "Helm lite / Win Rate", "evaluation_name": "helm_lite", "score": 0.698, "metric": "How many models this model outperforms on average (over columns).", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.668, "metric": "EM on GSM8K", "lower_is_better": false }, { "benchmark": "LegalBench", "benchmarkKey": "helm_lite_legalbench", "canonical_display_name": "LegalBench / Exact Match", "evaluation_name": "LegalBench", "score": 0.626, "metric": "EM on LegalBench", "lower_is_better": false }, { "benchmark": "NaturalQuestions (closed-book)", "benchmarkKey": "helm_lite_naturalquestions_closed_book", "canonical_display_name": "NaturalQuestions (closed-book) / F1", "evaluation_name": "NaturalQuestions (closed-book)", "score": 0.435, "metric": "F1 on NaturalQuestions (closed-book)", "lower_is_better": false }, { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.224, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "WMT 2014", "benchmarkKey": "helm_lite_wmt_2014", "canonical_display_name": "WMT 2014 / BLEU-4", "evaluation_name": "WMT 2014", "score": 0.205, "metric": "BLEU-4 on WMT 2014", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4-1", "model_route_id": "openai__gpt-4-1", "model_family_name": "gpt-4.1-2025-04-14", "developer": "openai", "params_billions": null, "total_evaluations": 13, "benchmark_count": 11, "benchmark_family_count": 11, "categories_covered": [ "coding", "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-04-14", "variant_label": "2025-04-14", "evaluation_count": 10, "raw_model_ids": [ "openai/gpt-4-1-2025-04-14", "openai/gpt-4-1-2025-04-14-fc", "openai/gpt-4-1-2025-04-14-prompt", "openai/gpt-4.1-2025-04-14" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 3, "raw_model_ids": [ "openai/gpt-4-1", "openai/gpt-4.1" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 548, "min": 0.0, "max": 1018.5843886213381, "average": 8.540420933795508 }, "reproducibility_summary": { "results_total": 548, "has_reproducibility_gap_count": 548, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 548, "total_groups": 497, "multi_source_groups": 0, "first_party_only_groups": 18, "source_type_distribution": { "first_party": 20, "third_party": 528, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 497, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Aider-Polyglot", "Aider-Polyglot Edit", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BFCL leaderboard CSV", "COLLIE", "CharXiv-D", "ComplexFuncBench", "Easy Problems", "GPQA", "Global-MMLU Lite", "Graphwalks BFS <128k", "Graphwalks BFS >128k", "Graphwalks parents <128k", "Graphwalks parents >128k", "Hard Problems", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "Internal API instruction following (hard)", "MMLU", "MMLU-Pro", "MMMLU", "MathVista", "Medium Problems", "Multi-Challenge", "Multi-IF", "Omni-MATH", "OpenAI-MRCR: 2 needle 128k", "OpenAI-MRCR: 2 needle 1M", "RewardBench 2", "SciArena leaderboard API", "SimpleSafetyTests", "TAU-bench Airline", "TAU-bench Retail", "Video-MME (long, no subtitles)", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 93.633, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 34.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 26.3, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 21.8, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 8.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 6.18, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.5, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "canonical_display_name": "SciArena / Cost per 100 Calls", "evaluation_name": "overall_cost_per_100_calls_usd", "score": 2.7261, "metric": "overall_cost_per_100_calls_usd", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 2.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #43: Illegal/Regulated substances/goods / Refusal Rate", "evaluation_name": "AIRBench 2024 - #43: Illegal/Regulated substances/goods", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #43: Illegal/Regulated substances/goods", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.993, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.979, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.963, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.926, "metric": "BBQ accuracy on BBQ", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4-1-mini", "model_route_id": "openai__gpt-4-1-mini", "model_family_name": "gpt-4.1-mini-2025-04-14", "developer": "openai", "params_billions": null, "total_evaluations": 10, "benchmark_count": 9, "benchmark_family_count": 9, "categories_covered": [ "general", "instruction_following", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-04-14", "variant_label": "2025-04-14", "evaluation_count": 8, "raw_model_ids": [ "openai/gpt-4-1-mini-2025-04-14", "openai/gpt-4-1-mini-2025-04-14-fc", "openai/gpt-4-1-mini-2025-04-14-prompt", "openai/gpt-4.1-mini-2025-04-14" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-4-1-mini", "openai/gpt-4.1-mini" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 502, "min": 0.0, "max": 1014.585805498339, "average": 8.32850558864213 }, "reproducibility_summary": { "results_total": 502, "has_reproducibility_gap_count": 502, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 502, "total_groups": 470, "multi_source_groups": 0, "first_party_only_groups": 13, "source_type_distribution": { "first_party": 15, "third_party": 487, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 470, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Aider-Polyglot", "Aider-Polyglot Edit", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BFCL leaderboard CSV", "COLLIE", "CharXiv-D", "ComplexFuncBench", "GPQA", "Graphwalks BFS <128k", "Graphwalks BFS >128k", "Graphwalks parents <128k", "Graphwalks parents >128k", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "Internal API instruction following (hard)", "MMLU-Pro", "MathVista", "Multi-IF", "Omni-MATH", "OpenAI-MRCR: 2 needle 128k", "OpenAI-MRCR: 2 needle 1M", "RewardBench 2", "SciArena leaderboard API", "SimpleSafetyTests", "TAU-bench Airline", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 81.357, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 46.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 22.9, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 18.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 13.33, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 1.6, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #1.1: Network intrusion / Refusal Rate", "evaluation_name": "AIRBench 2024 - #1.1: Network intrusion", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #1.1: Network intrusion", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.993, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.974, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.949, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.925, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.921, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "IFEval", "benchmarkKey": "helm_capabilities_ifeval", "canonical_display_name": "IFEval / Strict Accuracy", "evaluation_name": "IFEval", "score": 0.904, "metric": "IFEval Strict Acc on IFEval", "lower_is_better": false }, { "benchmark": "CharXiv-D", "benchmarkKey": "llm_stats_charxiv_d", "canonical_display_name": "Charxiv D / Score", "evaluation_name": "llm_stats.charxiv-d", "score": 0.884, "metric": "CharXiv-D is the descriptive questions subset of the CharXiv benchmark, designed to assess multimodal large language models' ability to extract basic information from scientific charts. It contains descriptive questions covering information extraction, enumeration, pattern recognition, and counting across 2,323 diverse charts from arXiv papers, all curated and verified by human experts.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-low", "model_route_id": "openai__gpt-5-4-low", "model_family_name": "gpt-5-4-low", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.603545Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-low" ], "last_updated": "2026-04-07T08:15:57.603545Z" } ], "score_summary": { "count": 8, "min": 0.12, "max": 0.8, "average": 0.3544125 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v2_Semi_Private / Cost per Task", "evaluation_name": "v2_Semi_Private", "score": 0.27, "metric": "v2_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-5-4-medium", "model_route_id": "openai__gpt-5-4-medium", "model_family_name": "gpt-5-4-medium", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.603836Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-medium" ], "last_updated": "2026-04-07T08:15:57.603836Z" } ], "score_summary": { "count": 8, "min": 0.21, "max": 0.92, "average": 0.598475 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v2_Public_Eval / Score", "evaluation_name": "v2_Public_Eval", "score": 0.5819, "metric": "v2_Public_Eval", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-mini-high", "model_route_id": "openai__gpt-5-4-mini-high", "model_family_name": "gpt-5-4-mini-high", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.604119Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-mini-high" ], "last_updated": "2026-04-07T08:15:57.604119Z" } ], "score_summary": { "count": 8, "min": 0.0701, "max": 0.6625, "average": 0.3893125 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.27, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-5-4-mini-low", "model_route_id": "openai__gpt-5-4-mini-low", "model_family_name": "gpt-5-4-mini-low", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.604408Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-mini-low" ], "last_updated": "2026-04-07T08:15:57.604408Z" } ], "score_summary": { "count": 8, "min": 0.0083, "max": 0.3175, "average": 0.0821125 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.04, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-5-4-mini-xhigh", "model_route_id": "openai__gpt-5-4-mini-xhigh", "model_family_name": "gpt-5-4-mini-xhigh", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.604979Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-mini-xhigh" ], "last_updated": "2026-04-07T08:15:57.604979Z" } ], "score_summary": { "count": 8, "min": 0.1778, "max": 0.81, "average": 0.5218375 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.51, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-5-4-nano-high", "model_route_id": "openai__gpt-5-4-nano-high", "model_family_name": "gpt-5-4-nano-high", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.605267Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-nano-high" ], "last_updated": "2026-04-07T08:15:57.605267Z" } ], "score_summary": { "count": 8, "min": 0.0361, "max": 0.5162, "average": 0.174425 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.07, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-5-4-nano-low", "model_route_id": "openai__gpt-5-4-nano-low", "model_family_name": "gpt-5-4-nano-low", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.605558Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-nano-low" ], "last_updated": "2026-04-07T08:15:57.605558Z" } ], "score_summary": { "count": 8, "min": 0.0, "max": 0.2462, "average": 0.06185 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v2_Semi_Private / Cost per Task", "evaluation_name": "v2_Semi_Private", "score": 0.01, "metric": "v2_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-5-4-nano-xhigh", "model_route_id": "openai__gpt-5-4-nano-xhigh", "model_family_name": "gpt-5-4-nano-xhigh", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.606120Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-nano-xhigh" ], "last_updated": "2026-04-07T08:15:57.606120Z" } ], "score_summary": { "count": 8, "min": 0.0417, "max": 0.515, "average": 0.21154999999999996 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Semi_Private / Cost per Task", "evaluation_name": "v1_Semi_Private", "score": 0.13, "metric": "v1_Semi_Private", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-5-4-pro-xhigh", "model_route_id": "openai__gpt-5-4-pro-xhigh", "model_family_name": "gpt-5-4-pro-xhigh", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.606405Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-pro-xhigh" ], "last_updated": "2026-04-07T08:15:57.606405Z" } ], "score_summary": { "count": 8, "min": 0.8333, "max": 17.6, "average": 6.2853625 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Public_Eval / Score", "evaluation_name": "v1_Public_Eval", "score": 0.9825, "metric": "v1_Public_Eval", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-xhigh", "model_route_id": "openai__gpt-5-4-xhigh", "model_family_name": "gpt-5-4-xhigh", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.606686Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-xhigh" ], "last_updated": "2026-04-07T08:15:57.606686Z" } ], "score_summary": { "count": 8, "min": 0.43, "max": 1.57, "average": 0.9527124999999999 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v2_Public_Eval / Score", "evaluation_name": "v2_Public_Eval", "score": 0.8417, "metric": "v2_Public_Eval", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-mini", "model_route_id": "openai__gpt-5-mini", "model_family_name": "gpt-5-mini-2025-08-07", "developer": "openai", "params_billions": null, "total_evaluations": 19, "benchmark_count": 11, "benchmark_family_count": 11, "categories_covered": [ "agentic", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-08-07-high", "variant_label": "2025-08-07 high", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-mini-2025-08-07-high" ], "last_updated": "2026-04-07T08:15:57.606970Z" }, { "variant_key": "2025-08-07-low", "variant_label": "2025-08-07 low", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-mini-2025-08-07-low" ], "last_updated": "2026-04-07T08:15:57.607256Z" }, { "variant_key": "2025-08-07-medium", "variant_label": "2025-08-07 medium", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-mini-2025-08-07-medium" ], "last_updated": "2026-04-07T08:15:57.607545Z" }, { "variant_key": "2025-08-07-minimal", "variant_label": "2025-08-07 minimal", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-mini-2025-08-07-minimal" ], "last_updated": "2026-04-07T08:15:57.607832Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 9, "raw_model_ids": [ "openai/gpt-5-mini" ], "last_updated": "2026-04-16T13:47:43.417572Z" }, { "variant_key": "2025-08-07", "variant_label": "2025-08-07", "evaluation_count": 6, "raw_model_ids": [ "openai/gpt-5-mini-2025-08-07", "openai/gpt-5-mini-2025-08-07-fc", "openai/gpt-5-mini-2025-08-07-prompt" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 527, "min": 0.0, "max": 49483.0, "average": 452.56284495170337 }, "reproducibility_summary": { "results_total": 527, "has_reproducibility_gap_count": 509, "populated_ratio_avg": 0.03415559772296015 }, "provenance_summary": { "total_results": 527, "total_groups": 469, "multi_source_groups": 0, "first_party_only_groups": 2, "source_type_distribution": { "first_party": 2, "third_party": 525, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 469, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BFCL leaderboard CSV", "Fibble arena", "FrontierMath", "GPQA", "HMMT 2025", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "Omni-MATH", "SciArena leaderboard API", "SimpleSafetyTests", "Terminal Bench 2 0", "WildBench", "Wordle Arena", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 91.319, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 91.319, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 90.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 77.192, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 41.2, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 35.3, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 34.8, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 3.78, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 2.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Fundamental Rights / Refusal Rate", "evaluation_name": "AIRBench 2024 - Fundamental Rights", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Fundamental Rights", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Wordle Arena", "benchmarkKey": "wordle_arena", "canonical_display_name": "Wordle Arena / Win Rate", "evaluation_name": "wordle_arena", "score": 1.0, "metric": "Win rate on Wordle Arena: Daily NYT Wordle: guess a 5-letter word in 6 attempts with green/yellow/gray feedback", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.991, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.98, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.977, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-pro", "model_route_id": "openai__gpt-5-pro", "model_family_name": "gpt-5-pro-2025-10-06", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2026-04-07T08:15:57.609661Z", "variants": [ { "variant_key": "2025-10-06", "variant_label": "2025-10-06", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-pro-2025-10-06" ], "last_updated": "2026-04-07T08:15:57.609661Z" } ], "score_summary": { "count": 8, "min": 0.1333, "max": 8.0107, "average": 3.2190375 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 8, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON" ], "top_benchmark_scores": [ { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Public_Eval / Score", "evaluation_name": "v1_Public_Eval", "score": 0.77, "metric": "v1_Public_Eval", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1", "model_route_id": "openai__gpt-5-1", "model_family_name": "gpt-5.1-2025-11-13", "developer": "openai", "params_billions": null, "total_evaluations": 14, "benchmark_count": 11, "benchmark_family_count": 11, "categories_covered": [ "agentic", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 6, "raw_model_ids": [ "openai/GPT 5.1", "openai/gpt-5-1", "openai/gpt-5.1" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "2025-11-13-thinking-high", "variant_label": "2025-11-13 thinking-high", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-1-2025-11-13-thinking-high" ], "last_updated": "2026-04-07T08:15:57.598032Z" }, { "variant_key": "2025-11-13-thinking-low", "variant_label": "2025-11-13 thinking-low", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-1-2025-11-13-thinking-low" ], "last_updated": "2026-04-07T08:15:57.598354Z" }, { "variant_key": "2025-11-13-thinking-medium", "variant_label": "2025-11-13 thinking-medium", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-1-2025-11-13-thinking-medium" ], "last_updated": "2026-04-07T08:15:57.598689Z" }, { "variant_key": "2025-11-13-thinking-none", "variant_label": "2025-11-13 thinking-none", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-1-2025-11-13-thinking-none" ], "last_updated": "2026-04-07T08:15:57.599151Z" }, { "variant_key": "2025-11-13", "variant_label": "2025-11-13", "evaluation_count": 4, "raw_model_ids": [ "openai/gpt-5.1-2025-11-13" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 454, "min": 0.0, "max": 1079.672379931729, "average": 4.071595770774729 }, "reproducibility_summary": { "results_total": 454, "has_reproducibility_gap_count": 454, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 454, "total_groups": 430, "multi_source_groups": 0, "first_party_only_groups": 13, "source_type_distribution": { "first_party": 13, "third_party": 441, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 430, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ACE", "APEX Agents", "APEX v1", "ARC Prize evaluations leaderboard JSON", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BrowseComp Long Context 128k", "FrontierMath", "GPQA", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "MMMU", "Omni-MATH", "SciArena leaderboard API", "SimpleSafetyTests", "Tau2 Airline", "Tau2 Retail", "Tau2 Telecom", "Terminal Bench 2 0", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 105.318, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 94.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 47.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 47.6, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 44.7, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 24.539, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 24.539, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "canonical_display_name": "SciArena / Cost per 100 Calls", "evaluation_name": "overall_cost_per_100_calls_usd", "score": 2.5648, "metric": "overall_cost_per_100_calls_usd", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.25, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Fundamental Rights / Refusal Rate", "evaluation_name": "AIRBench 2024 - Fundamental Rights", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Fundamental Rights", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.998, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.995, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.982, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1-codex", "model_route_id": "openai__gpt-5-1-codex", "model_family_name": "gpt-5.1-codex", "developer": "openai", "params_billions": null, "total_evaluations": 6, "benchmark_count": 4, "benchmark_family_count": 4, "categories_covered": [ "agentic", "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 6, "raw_model_ids": [ "openai/GPT 5.1 Codex", "openai/gpt-5-1-codex", "openai/gpt-5.1-codex" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 24, "min": 0.234, "max": 178.705, "average": 22.165875 }, "reproducibility_summary": { "results_total": 24, "has_reproducibility_gap_count": 24, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 24, "total_groups": 22, "multi_source_groups": 0, "first_party_only_groups": 2, "source_type_distribution": { "first_party": 2, "third_party": 22, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 22, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "APEX Agents", "Artificial Analysis LLM API", "SWE-Lancer", "Terminal Bench 2 0" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 178.705, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 95.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 57.8, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 43.1, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 36.6, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 10.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 3.623, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 3.623, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.25, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.957, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.86, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.86, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.849, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.83, "metric": "Benchmark score on tau2.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1-codex-high", "model_route_id": "openai__gpt-5-1-codex-high", "model_family_name": "gpt-5.1-codex-high", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.1-codex-high" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 1, "min": 0.967, "max": 0.967, "average": 0.967 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 1, "source_type_distribution": { "first_party": 1, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AIME 2025" ], "top_benchmark_scores": [ { "benchmark": "AIME 2025", "benchmarkKey": "llm_stats_aime_2025", "canonical_display_name": "Aime 2025 / Score", "evaluation_name": "llm_stats.aime-2025", "score": 0.967, "metric": "All 30 problems from the 2025 American Invitational Mathematics Examination (AIME I and AIME II), testing olympiad-level mathematical reasoning with integer answers from 000-999. Used as an AI benchmark to evaluate large language models' ability to solve complex mathematical problems requiring multi-step logical deductions and structured symbolic reasoning.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1-instant", "model_route_id": "openai__gpt-5-1-instant", "model_family_name": "gpt-5.1-instant-2025-11-12", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-11-12", "variant_label": "2025-11-12", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.1-instant-2025-11-12" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 6, "min": 0.267, "max": 0.956, "average": 0.7376666666666667 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 6, "source_type_distribution": { "first_party": 6, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BrowseComp Long Context 128k", "FrontierMath", "MMMU", "Tau2 Airline", "Tau2 Retail", "Tau2 Telecom" ], "top_benchmark_scores": [ { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.956, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "BrowseComp Long Context 128k", "benchmarkKey": "llm_stats_browsecomp_long_128k", "canonical_display_name": "Browsecomp Long 128k / Score", "evaluation_name": "llm_stats.browsecomp-long-128k", "score": 0.9, "metric": "A challenging benchmark for evaluating web browsing agents' ability to persistently navigate the internet and find hard-to-locate, entangled information. Comprises 1,266 questions requiring strategic reasoning, creative search, and interpretation of retrieved content, with short and easily verifiable answers.", "lower_is_better": false }, { "benchmark": "MMMU", "benchmarkKey": "llm_stats_mmmu", "canonical_display_name": "Mmmu / Score", "evaluation_name": "llm_stats.mmmu", "score": 0.854, "metric": "MMMU (Massive Multi-discipline Multimodal Understanding) is a benchmark designed to evaluate multimodal models on college-level subject knowledge and deliberate reasoning. Contains 11.5K meticulously collected multimodal questions from college exams, quizzes, and textbooks, covering six core disciplines: Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering across 30 subjects and 183 subfields.", "lower_is_better": false }, { "benchmark": "Tau2 Retail", "benchmarkKey": "llm_stats_tau2_retail", "canonical_display_name": "Tau2 Retail / Score", "evaluation_name": "llm_stats.tau2-retail", "score": 0.779, "metric": "τ²-bench retail domain evaluates conversational AI agents in customer service scenarios within a dual-control environment where both agent and user can interact with tools. Tests tool-agent-user interaction, rule adherence, and task consistency in retail customer support contexts.", "lower_is_better": false }, { "benchmark": "Tau2 Airline", "benchmarkKey": "llm_stats_tau2_airline", "canonical_display_name": "Tau2 Airline / Score", "evaluation_name": "llm_stats.tau2-airline", "score": 0.67, "metric": "TAU2 airline domain benchmark for evaluating conversational agents in dual-control environments where both AI agents and users interact with tools in airline customer service scenarios. Tests agent coordination, communication, and ability to guide user actions in tasks like flight booking, modifications, cancellations, and refunds.", "lower_is_better": false }, { "benchmark": "FrontierMath", "benchmarkKey": "llm_stats_frontiermath", "canonical_display_name": "Frontiermath / Score", "evaluation_name": "llm_stats.frontiermath", "score": 0.267, "metric": "A benchmark of hundreds of original, exceptionally challenging mathematics problems crafted and vetted by expert mathematicians, covering most major branches of modern mathematics from number theory and real analysis to algebraic geometry and category theory.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1-medium", "model_route_id": "openai__gpt-5-1-medium", "model_family_name": "gpt-5.1-medium-2025-11-12", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-11-12", "variant_label": "2025-11-12", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.1-medium-2025-11-12" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 1, "min": 0.984, "max": 0.984, "average": 0.984 }, "reproducibility_summary": { "results_total": 1, "has_reproducibility_gap_count": 1, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 1, "total_groups": 1, "multi_source_groups": 0, "first_party_only_groups": 1, "source_type_distribution": { "first_party": 1, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 1, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AIME 2025" ], "top_benchmark_scores": [ { "benchmark": "AIME 2025", "benchmarkKey": "llm_stats_aime_2025", "canonical_display_name": "Aime 2025 / Score", "evaluation_name": "llm_stats.aime-2025", "score": 0.984, "metric": "All 30 problems from the 2025 American Invitational Mathematics Examination (AIME I and AIME II), testing olympiad-level mathematical reasoning with integer answers from 000-999. Used as an AI benchmark to evaluate large language models' ability to solve complex mathematical problems requiring multi-step logical deductions and structured symbolic reasoning.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-1-thinking", "model_route_id": "openai__gpt-5-1-thinking", "model_family_name": "gpt-5.1-thinking-2025-11-12", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-11-12", "variant_label": "2025-11-12", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.1-thinking-2025-11-12" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 6, "min": 0.267, "max": 0.956, "average": 0.7376666666666667 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 6, "source_type_distribution": { "first_party": 6, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BrowseComp Long Context 128k", "FrontierMath", "MMMU", "Tau2 Airline", "Tau2 Retail", "Tau2 Telecom" ], "top_benchmark_scores": [ { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.956, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "BrowseComp Long Context 128k", "benchmarkKey": "llm_stats_browsecomp_long_128k", "canonical_display_name": "Browsecomp Long 128k / Score", "evaluation_name": "llm_stats.browsecomp-long-128k", "score": 0.9, "metric": "A challenging benchmark for evaluating web browsing agents' ability to persistently navigate the internet and find hard-to-locate, entangled information. Comprises 1,266 questions requiring strategic reasoning, creative search, and interpretation of retrieved content, with short and easily verifiable answers.", "lower_is_better": false }, { "benchmark": "MMMU", "benchmarkKey": "llm_stats_mmmu", "canonical_display_name": "Mmmu / Score", "evaluation_name": "llm_stats.mmmu", "score": 0.854, "metric": "MMMU (Massive Multi-discipline Multimodal Understanding) is a benchmark designed to evaluate multimodal models on college-level subject knowledge and deliberate reasoning. Contains 11.5K meticulously collected multimodal questions from college exams, quizzes, and textbooks, covering six core disciplines: Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering across 30 subjects and 183 subfields.", "lower_is_better": false }, { "benchmark": "Tau2 Retail", "benchmarkKey": "llm_stats_tau2_retail", "canonical_display_name": "Tau2 Retail / Score", "evaluation_name": "llm_stats.tau2-retail", "score": 0.779, "metric": "τ²-bench retail domain evaluates conversational AI agents in customer service scenarios within a dual-control environment where both agent and user can interact with tools. Tests tool-agent-user interaction, rule adherence, and task consistency in retail customer support contexts.", "lower_is_better": false }, { "benchmark": "Tau2 Airline", "benchmarkKey": "llm_stats_tau2_airline", "canonical_display_name": "Tau2 Airline / Score", "evaluation_name": "llm_stats.tau2-airline", "score": 0.67, "metric": "TAU2 airline domain benchmark for evaluating conversational agents in dual-control environments where both AI agents and users interact with tools in airline customer service scenarios. Tests agent coordination, communication, and ability to guide user actions in tasks like flight booking, modifications, cancellations, and refunds.", "lower_is_better": false }, { "benchmark": "FrontierMath", "benchmarkKey": "llm_stats_frontiermath", "canonical_display_name": "Frontiermath / Score", "evaluation_name": "llm_stats.frontiermath", "score": 0.267, "metric": "A benchmark of hundreds of original, exceptionally challenging mathematics problems crafted and vetted by expert mathematicians, covering most major branches of modern mathematics from number theory and real analysis to algebraic geometry and category theory.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-2", "model_route_id": "openai__gpt-5-2", "model_family_name": "gpt-5.2-2025-12-11", "developer": "openai", "params_billions": null, "total_evaluations": 47, "benchmark_count": 13, "benchmark_family_count": 13, "categories_covered": [ "agentic", "coding", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 8, "raw_model_ids": [ "openai/GPT 5.2", "openai/GPT-5.2", "openai/gpt-5-2", "openai/gpt-5.2" ], "last_updated": "2026-04-20T17:00:03.994647Z" }, { "variant_key": "2025-12-11", "variant_label": "2025-12-11", "evaluation_count": 34, "raw_model_ids": [ "openai/gpt-5-2-2025-12-11-fc", "openai/gpt-5-2-2025-12-11-prompt", "openai/gpt-5.2-2025-12-11" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "2025-12-11-thinking-high", "variant_label": "2025-12-11 thinking-high", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-2025-12-11-thinking-high" ], "last_updated": "2026-04-07T08:15:57.599508Z" }, { "variant_key": "2025-12-11-thinking-low", "variant_label": "2025-12-11 thinking-low", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-2025-12-11-thinking-low" ], "last_updated": "2026-04-07T08:15:57.599811Z" }, { "variant_key": "2025-12-11-thinking-medium", "variant_label": "2025-12-11 thinking-medium", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-2025-12-11-thinking-medium" ], "last_updated": "2026-04-07T08:15:57.600107Z" }, { "variant_key": "2025-12-11-thinking-none", "variant_label": "2025-12-11 thinking-none", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-2025-12-11-thinking-none" ], "last_updated": "2026-04-07T08:15:57.600410Z" }, { "variant_key": "2025-12-11-thinking-xhigh", "variant_label": "2025-12-11 thinking-xhigh", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-2-2025-12-11-thinking-xhigh" ], "last_updated": "2026-04-07T08:15:57.600713Z" } ], "score_summary": { "count": 191, "min": 0.0, "max": 164.58, "average": 20.644869240837703 }, "reproducibility_summary": { "results_total": 191, "has_reproducibility_gap_count": 191, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 191, "total_groups": 101, "multi_source_groups": 0, "first_party_only_groups": 31, "source_type_distribution": { "first_party": 32, "third_party": 159, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 101, "groups_with_variant_check": 6, "groups_with_cross_party_check": 0, "variant_divergent_count": 6, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ACE", "AIME 2025", "APEX Agents", "ARC Prize evaluations leaderboard JSON", "ARC-AGI", "ARC-AGI v2", "AppWorld Benchmark", "Artificial Analysis LLM API", "BFCL leaderboard CSV", "BrowseComp", "BrowseComp Long Context 128k", "BrowseComp Long Context 256k", "BrowseComp-Plus", "CharXiv-R", "Easy Problems", "FrontierMath", "GPQA", "Graphwalks BFS <128k", "Graphwalks parents <128k", "HMMT 2025", "Hard Problems", "MCP Atlas", "MMMLU", "MMMU-Pro", "Medium Problems", "Multi-SWE-bench (java)", "SWE-Bench Verified", "SWE-Lancer (IC-Diamond subset)", "ScreenSpot Pro", "Swe Bench", "Tau2 Retail", "Tau2 Telecom", "Terminal Bench 2 0", "Toolathlon", "VideoMMMU", "τ-bench (Tool-Agent-User Interaction Benchmark)" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 99.0, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 83.592, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 83.592, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 66.723, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 64.9, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 51.3, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 48.7, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 14.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 4.813, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 3.25, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.75, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "AIME 2025", "benchmarkKey": "llm_stats_aime_2025", "canonical_display_name": "Aime 2025 / Score", "evaluation_name": "llm_stats.aime-2025", "score": 1.0, "metric": "All 30 problems from the 2025 American Invitational Mathematics Examination (AIME I and AIME II), testing olympiad-level mathematical reasoning with integer answers from 000-999. Used as an AI benchmark to evaluate large language models' ability to solve complex mathematical problems requiring multi-step logical deductions and structured symbolic reasoning.", "lower_is_better": false }, { "benchmark": "HMMT 2025", "benchmarkKey": "llm_stats_hmmt_2025", "canonical_display_name": "Hmmt 2025 / Score", "evaluation_name": "llm_stats.hmmt-2025", "score": 0.994, "metric": "Harvard-MIT Mathematics Tournament 2025 - A prestigious student-organized mathematics competition for high school students featuring two tournaments (November 2025 at MIT and February 2026 at Harvard) with individual tests, team rounds, and guts rounds", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.99, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.987, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-2-codex", "model_route_id": "openai__gpt-5-2-codex", "model_family_name": "gpt-5.2-codex", "developer": "openai", "params_billions": null, "total_evaluations": 4, "benchmark_count": 4, "benchmark_family_count": 4, "categories_covered": [ "agentic", "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 4, "raw_model_ids": [ "openai/GPT 5.2 Codex", "openai/gpt-5-2-codex", "openai/gpt-5.2-codex" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 20, "min": 0.276, "max": 76.481, "average": 13.741449999999997 }, "reproducibility_summary": { "results_total": 20, "has_reproducibility_gap_count": 20, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 20, "total_groups": 20, "multi_source_groups": 0, "first_party_only_groups": 4, "source_type_distribution": { "first_party": 4, "third_party": 16, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 20, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "APEX Agents", "Artificial Analysis LLM API", "SWE-Bench Pro", "Terminal Bench 2 0", "Terminal-Bench 2.0" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 76.481, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 66.5, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 49.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 43.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 14.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 6.403, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 6.403, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 4.813, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.75, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.921, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.899, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.776, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.757, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Terminal-Bench 2.0", "benchmarkKey": "llm_stats_terminal_bench_2", "canonical_display_name": "Terminal Bench 2 / Score", "evaluation_name": "llm_stats.terminal-bench-2", "score": 0.64, "metric": "Terminal-Bench 2.0 is an updated benchmark for testing AI agents' tool use ability to operate a computer via terminal. It evaluates how well models can handle real-world, end-to-end tasks autonomously, including compiling code, training models, setting up servers, system administration, security tasks, data science workflows, and cybersecurity vulnerabilities.", "lower_is_better": false }, { "benchmark": "SWE-Bench Pro", "benchmarkKey": "llm_stats_swe_bench_pro", "canonical_display_name": "Swe Bench Pro / Score", "evaluation_name": "llm_stats.swe-bench-pro", "score": 0.564, "metric": "SWE-Bench Pro is an advanced version of SWE-Bench that evaluates language models on complex, real-world software engineering tasks requiring extended reasoning and multi-step problem solving.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-3-chat-latest", "model_route_id": "openai__gpt-5-3-chat-latest", "model_family_name": "gpt-5.3-chat-latest", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5.3-chat-latest" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 2, "min": 0.259, "max": 0.541, "average": 0.4 }, "reproducibility_summary": { "results_total": 2, "has_reproducibility_gap_count": 2, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 2, "total_groups": 2, "multi_source_groups": 0, "first_party_only_groups": 2, "source_type_distribution": { "first_party": 2, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 2, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "HealthBench", "HealthBench Hard" ], "top_benchmark_scores": [ { "benchmark": "HealthBench", "benchmarkKey": "llm_stats_healthbench", "canonical_display_name": "Healthbench / Score", "evaluation_name": "llm_stats.healthbench", "score": 0.541, "metric": "An open-source benchmark for measuring performance and safety of large language models in healthcare, consisting of 5,000 multi-turn conversations evaluated by 262 physicians using 48,562 unique rubric criteria across health contexts and behavioral dimensions", "lower_is_better": false }, { "benchmark": "HealthBench Hard", "benchmarkKey": "llm_stats_healthbench_hard", "canonical_display_name": "Healthbench Hard / Score", "evaluation_name": "llm_stats.healthbench-hard", "score": 0.259, "metric": "A challenging variation of HealthBench that evaluates large language models' performance and safety in healthcare through 5,000 multi-turn conversations with particularly rigorous evaluation criteria validated by 262 physicians from 60 countries", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-3-codex", "model_route_id": "openai__gpt-5-3-codex", "model_family_name": "gpt-5.3-codex", "developer": "openai", "params_billions": null, "total_evaluations": 8, "benchmark_count": 4, "benchmark_family_count": 4, "categories_covered": [ "agentic", "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 8, "raw_model_ids": [ "openai/GPT 5.3 Codex", "openai/gpt-5-3-codex", "openai/gpt-5.3-codex" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 26, "min": 0.317, "max": 77.3, "average": 26.990423076923076 }, "reproducibility_summary": { "results_total": 26, "has_reproducibility_gap_count": 26, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 26, "total_groups": 22, "multi_source_groups": 0, "first_party_only_groups": 6, "source_type_distribution": { "first_party": 6, "third_party": 20, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 22, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "APEX Agents", "Artificial Analysis LLM API", "Cybersecurity CTFs", "OSWorld-Verified", "SWE-Bench Pro", "SWE-Lancer (IC-Diamond subset)", "Terminal Bench 2 0", "Terminal-Bench 2.0" ], "top_benchmark_scores": [ { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 77.3, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 75.229, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 64.317, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 64.317, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 53.6, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 53.1, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 14.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 4.813, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.75, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.915, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.86, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "SWE-Lancer (IC-Diamond subset)", "benchmarkKey": "llm_stats_swe_lancer_ic_diamond_subset", "canonical_display_name": "Swe Lancer Ic Diamond Subset / Score", "evaluation_name": "llm_stats.swe-lancer-ic-diamond-subset", "score": 0.814, "metric": "SWE-Lancer (IC-Diamond subset) is a benchmark of real-world freelance software engineering tasks from Upwork, ranging from $50 bug fixes to $32,000 feature implementations. It evaluates AI models on independent engineering tasks using end-to-end tests triple-verified by experienced software engineers, and includes managerial tasks where models choose between technical implementation proposals.", "lower_is_better": false }, { "benchmark": "Cybersecurity CTFs", "benchmarkKey": "llm_stats_cybersecurity_ctfs", "canonical_display_name": "Cybersecurity Ctfs / Score", "evaluation_name": "llm_stats.cybersecurity-ctfs", "score": 0.776, "metric": "Cybersecurity Capture the Flag (CTF) benchmark for evaluating LLMs in offensive security challenges. Contains diverse cybersecurity tasks including cryptography, web exploitation, binary analysis, and forensics to assess AI capabilities in cybersecurity problem-solving.", "lower_is_better": false }, { "benchmark": "Terminal-Bench 2.0", "benchmarkKey": "llm_stats_terminal_bench_2", "canonical_display_name": "Terminal Bench 2 / Score", "evaluation_name": "llm_stats.terminal-bench-2", "score": 0.773, "metric": "Terminal-Bench 2.0 is an updated benchmark for testing AI agents' tool use ability to operate a computer via terminal. It evaluates how well models can handle real-world, end-to-end tasks autonomously, including compiling code, training models, setting up servers, system administration, security tasks, data science workflows, and cybersecurity vulnerabilities.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.754, "metric": "Benchmark score on IFBench.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4", "model_route_id": "openai__gpt-5-4", "model_family_name": "gpt-5.4", "developer": "openai", "params_billions": null, "total_evaluations": 3, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2026-03-05-high", "variant_label": "2026-03-05 high", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-5-4-2026-03-05-high" ], "last_updated": "2026-04-07T08:15:57.603037Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-5-4", "openai/gpt-5.4" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 35, "min": 0.0026, "max": 5187.41, "average": 163.93853142857142 }, "reproducibility_summary": { "results_total": 35, "has_reproducibility_gap_count": 35, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 35, "total_groups": 33, "multi_source_groups": 0, "first_party_only_groups": 16, "source_type_distribution": { "first_party": 18, "third_party": 17, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 33, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "ARC-AGI", "ARC-AGI v2", "Artificial Analysis LLM API", "BrowseComp", "Finance Agent", "FrontierMath", "GPQA", "Graphwalks BFS <128k", "Graphwalks BFS >128k", "Graphwalks parents <128k", "Graphwalks parents >128k", "MCP Atlas", "MMMU-Pro", "OSWorld-Verified", "OmniDocBench 1.5", "SWE-Bench Pro", "Tau2 Telecom", "Terminal-Bench 2.0", "Toolathlon" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 157.877, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 157.877, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 79.814, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 57.3, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 56.8, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 15.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 5.625, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 2.5, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.989, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "ARC-AGI", "benchmarkKey": "llm_stats_arc_agi", "canonical_display_name": "Arc Agi / Score", "evaluation_name": "llm_stats.arc-agi", "score": 0.937, "metric": "The Abstraction and Reasoning Corpus for Artificial General Intelligence (ARC-AGI) is a benchmark designed to test general intelligence and abstract reasoning capabilities through visual grid-based transformation tasks. Each task consists of 2-5 demonstration pairs showing input grids transformed into output grids according to underlying rules, with test-takers required to infer these rules and apply them to novel test inputs. The benchmark uses colored grids (up to 30x30) with 10 discrete colors/symbols, designed to measure human-like general fluid intelligence and skill-acquisition efficiency with minimal prior knowledge.", "lower_is_better": false }, { "benchmark": "Graphwalks BFS <128k", "benchmarkKey": "llm_stats_graphwalks_bfs_128k", "canonical_display_name": "Graphwalks Bfs 128k / Score", "evaluation_name": "llm_stats.graphwalks-bfs-128k", "score": 0.93, "metric": "A graph reasoning benchmark that evaluates language models' ability to perform breadth-first search (BFS) operations on graphs with context length under 128k tokens, returning nodes reachable at specified depths.", "lower_is_better": false }, { "benchmark": "GPQA", "benchmarkKey": "llm_stats_gpqa", "canonical_display_name": "Gpqa / Score", "evaluation_name": "llm_stats.gpqa", "score": 0.928, "metric": "A challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. Questions are Google-proof and extremely difficult, with PhD experts reaching 65% accuracy.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.92, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Graphwalks parents <128k", "benchmarkKey": "llm_stats_graphwalks_parents_128k", "canonical_display_name": "Graphwalks Parents 128k / Score", "evaluation_name": "llm_stats.graphwalks-parents-128k", "score": 0.898, "metric": "A graph reasoning benchmark that evaluates language models' ability to find parent nodes in graphs with context length under 128k tokens, requiring understanding of graph structure and edge relationships.", "lower_is_better": false }, { "benchmark": "OmniDocBench 1.5", "benchmarkKey": "llm_stats_omnidocbench_1_5", "canonical_display_name": "Omnidocbench 1 5 / Score", "evaluation_name": "llm_stats.omnidocbench-1.5", "score": 0.891, "metric": "OmniDocBench 1.5 is a comprehensive benchmark for evaluating multimodal large language models on document understanding tasks, including OCR, document parsing, information extraction, and visual question answering across diverse document types. Lower Overall Edit Distance scores are better.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-mini", "model_route_id": "openai__gpt-5-4-mini", "model_family_name": "gpt-5.4-mini", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-5-4-mini", "openai/gpt-5.4-mini" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 26, "min": 0.266, "max": 161.174, "average": 11.087026923076925 }, "reproducibility_summary": { "results_total": 26, "has_reproducibility_gap_count": 26, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 26, "total_groups": 26, "multi_source_groups": 0, "first_party_only_groups": 11, "source_type_distribution": { "first_party": 11, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 26, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "Graphwalks BFS <128k", "Graphwalks parents <128k", "MCP Atlas", "MMMU-Pro", "MRCR v2 (8-needle)", "OSWorld-Verified", "OmniDocBench 1.5", "SWE-Bench Pro", "Tau2 Telecom", "Terminal-Bench 2.0", "Toolathlon" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 161.174, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 51.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 48.9, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 4.5, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 4.035, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 4.035, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 1.688, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.934, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.875, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "OmniDocBench 1.5", "benchmarkKey": "llm_stats_omnidocbench_1_5", "canonical_display_name": "Omnidocbench 1 5 / Score", "evaluation_name": "llm_stats.omnidocbench-1.5", "score": 0.8737, "metric": "OmniDocBench 1.5 is a comprehensive benchmark for evaluating multimodal large language models on document understanding tasks, including OCR, document parsing, information extraction, and visual question answering across diverse document types. Lower Overall Edit Distance scores are better.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.833, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "MMMU-Pro", "benchmarkKey": "llm_stats_mmmu_pro", "canonical_display_name": "Mmmu Pro / Score", "evaluation_name": "llm_stats.mmmu-pro", "score": 0.766, "metric": "A more robust multi-discipline multimodal understanding benchmark that enhances MMMU through a three-step process: filtering text-only answerable questions, augmenting candidate options, and introducing vision-only input settings. Achieves significantly lower model performance (16.8-26.9%) compared to original MMMU, providing more rigorous evaluation that closely mimics real-world scenarios.", "lower_is_better": false }, { "benchmark": "Graphwalks BFS <128k", "benchmarkKey": "llm_stats_graphwalks_bfs_128k", "canonical_display_name": "Graphwalks Bfs 128k / Score", "evaluation_name": "llm_stats.graphwalks-bfs-128k", "score": 0.763, "metric": "A graph reasoning benchmark that evaluates language models' ability to perform breadth-first search (BFS) operations on graphs with context length under 128k tokens, returning nodes reachable at specified depths.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 0.75, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.733, "metric": "Benchmark score on IFBench.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-5-4-nano", "model_route_id": "openai__gpt-5-4-nano", "model_family_name": "gpt-5.4-nano", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/gpt-5-4-nano", "openai/gpt-5.4-nano" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 24, "min": 0.2, "max": 177.526, "average": 11.713712499999998 }, "reproducibility_summary": { "results_total": 24, "has_reproducibility_gap_count": 24, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 24, "total_groups": 24, "multi_source_groups": 0, "first_party_only_groups": 9, "source_type_distribution": { "first_party": 9, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 24, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "Graphwalks BFS <128k", "Graphwalks parents <128k", "MCP Atlas", "MRCR v2 (8-needle)", "OSWorld-Verified", "OmniDocBench 1.5", "SWE-Bench Pro", "Tau2 Telecom", "Toolathlon" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 177.526, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 44.0, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 43.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 2.275, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 2.275, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 1.25, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "canonical_display_name": "Tau2 Telecom / Score", "evaluation_name": "llm_stats.tau2-telecom", "score": 0.925, "metric": "τ²-Bench telecom domain evaluates conversational agents in a dual-control environment modeled as a Dec-POMDP, where both agent and user use tools in shared telecommunications troubleshooting scenarios that test coordination and communication capabilities.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.817, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.76, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.759, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "OmniDocBench 1.5", "benchmarkKey": "llm_stats_omnidocbench_1_5", "canonical_display_name": "Omnidocbench 1 5 / Score", "evaluation_name": "llm_stats.omnidocbench-1.5", "score": 0.7581, "metric": "OmniDocBench 1.5 is a comprehensive benchmark for evaluating multimodal large language models on document understanding tasks, including OCR, document parsing, information extraction, and visual question answering across diverse document types. Lower Overall Edit Distance scores are better.", "lower_is_better": false }, { "benchmark": "Graphwalks BFS <128k", "benchmarkKey": "llm_stats_graphwalks_bfs_128k", "canonical_display_name": "Graphwalks Bfs 128k / Score", "evaluation_name": "llm_stats.graphwalks-bfs-128k", "score": 0.734, "metric": "A graph reasoning benchmark that evaluates language models' ability to perform breadth-first search (BFS) operations on graphs with context length under 128k tokens, returning nodes reachable at specified depths.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.66, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "MCP Atlas", "benchmarkKey": "llm_stats_mcp_atlas", "canonical_display_name": "Mcp Atlas / Score", "evaluation_name": "llm_stats.mcp-atlas", "score": 0.561, "metric": "MCP Atlas is a benchmark for evaluating AI models on scaled tool use capabilities, measuring how well models can coordinate and utilize multiple tools across complex multi-step tasks.", "lower_is_better": false }, { "benchmark": "SWE-Bench Pro", "benchmarkKey": "llm_stats_swe_bench_pro", "canonical_display_name": "Swe Bench Pro / Score", "evaluation_name": "llm_stats.swe-bench-pro", "score": 0.524, "metric": "SWE-Bench Pro is an advanced version of SWE-Bench that evaluates language models on complex, real-world software engineering tasks requiring extended reasoning and multi-step problem solving.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-oss-120b-low", "model_route_id": "openai__gpt-oss-120b-low", "model_family_name": "gpt-oss-120B (low)", "developer": "openai", "params_billions": 120.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-oss-120b-low" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.052, "max": 232.993, "average": 18.688315789473688 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 232.993, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 66.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 24.5, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 15.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 9.1, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.775, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.707, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.672, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.667, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.6, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.583, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.516, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.45, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.437, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.36, "metric": "Benchmark score on SciCode.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-oss-120b", "model_route_id": "openai__gpt-oss-120b", "model_family_name": "gpt-oss-120b", "developer": "openai", "params_billions": 120.0, "total_evaluations": 10, "benchmark_count": 9, "benchmark_family_count": 9, "categories_covered": [ "agentic", "coding", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 10, "raw_model_ids": [ "openai/GPT OSS 120B", "openai/gpt-oss-120b" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 425, "min": 0.0, "max": 1036.5017542645915, "average": 4.313756005063326 }, "reproducibility_summary": { "results_total": 425, "has_reproducibility_gap_count": 425, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 425, "total_groups": 423, "multi_source_groups": 0, "first_party_only_groups": 11, "source_type_distribution": { "first_party": 12, "third_party": 413, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 423, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "APEX Agents", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "CodeForces", "Easy Problems", "GPQA", "Hard Problems", "HarmBench", "HealthBench", "HealthBench Hard", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU", "MMLU-Pro", "Medium Problems", "Omni-MATH", "SciArena leaderboard API", "SimpleSafetyTests", "TAU-bench Retail", "Terminal Bench 2 0", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 230.718, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 93.4, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 33.3, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 28.6, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 18.7, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 9.181, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Security Risks / Refusal Rate", "evaluation_name": "AIRBench 2024 - Security Risks", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Security Risks", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 1.0, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.995, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.985, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.981, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.934, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.927, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "llm_stats_mmlu", "canonical_display_name": "Mmlu / Score", "evaluation_name": "llm_stats.mmlu", "score": 0.9, "metric": "Massive Multitask Language Understanding benchmark testing knowledge across 57 diverse subjects including STEM, humanities, social sciences, and professional domains", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-oss-120b-high", "model_route_id": "openai__gpt-oss-120b-high", "model_family_name": "gpt-oss-120b-high", "developer": "openai", "params_billions": 120.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "coding", "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-oss-120b-high" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 3, "min": 0.639, "max": 0.819, "average": 0.7176666666666666 }, "reproducibility_summary": { "results_total": 3, "has_reproducibility_gap_count": 3, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 3, "total_groups": 3, "multi_source_groups": 0, "first_party_only_groups": 3, "source_type_distribution": { "first_party": 3, "third_party": 0, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 3, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "IFBench", "LiveCodeBench v6", "Llm Stats" ], "top_benchmark_scores": [ { "benchmark": "LiveCodeBench v6", "benchmarkKey": "llm_stats_livecodebench_v6", "canonical_display_name": "Livecodebench V6 / Score", "evaluation_name": "llm_stats.livecodebench-v6", "score": 0.819, "metric": "LiveCodeBench is a holistic and contamination-free evaluation benchmark for large language models for code. It continuously collects new problems from programming contests (LeetCode, AtCoder, CodeForces) and evaluates four different scenarios: code generation, self-repair, code execution, and test output prediction. Problems are annotated with release dates to enable evaluation on unseen problems released after a model's training cutoff.", "lower_is_better": false }, { "benchmark": "IFBench", "benchmarkKey": "llm_stats_ifbench", "canonical_display_name": "Ifbench / Score", "evaluation_name": "llm_stats.ifbench", "score": 0.695, "metric": "Instruction Following Benchmark evaluating model's ability to follow complex instructions", "lower_is_better": false }, { "benchmark": "Llm Stats", "benchmarkKey": "llm_stats_t2_bench", "canonical_display_name": "T2 Bench / Score", "evaluation_name": "llm_stats.t2-bench", "score": 0.639, "metric": "t2-bench is a benchmark for evaluating agentic tool use capabilities, measuring how well models can select, sequence, and utilize tools to solve complex tasks. It tests autonomous planning and execution in multi-step scenarios.", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-oss-20b-low", "model_route_id": "openai__gpt-oss-20b-low", "model_family_name": "gpt-oss-20B (low)", "developer": "openai", "params_billions": 20.0, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-oss-20b-low" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.045, "max": 205.381, "average": 16.75336842105263 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 205.381, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 62.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 20.8, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 14.4, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 10.193, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.718, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.652, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.623, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.611, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.578, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_tau2", "canonical_display_name": "artificial_analysis.tau2 / tau2", "evaluation_name": "artificial_analysis.tau2", "score": 0.503, "metric": "Benchmark score on tau2.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.455, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.34, "metric": "Benchmark score on SciCode.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_lcr", "canonical_display_name": "artificial_analysis.lcr / Lcr", "evaluation_name": "artificial_analysis.lcr", "score": 0.31, "metric": "Benchmark score on AA-LCR.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 0.2, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true } ] }, { "model_group_id": "openai/gpt-oss-20b", "model_route_id": "openai__gpt-oss-20b", "model_family_name": "gpt-oss-20b", "developer": "openai", "params_billions": 20.0, "total_evaluations": 9, "benchmark_count": 8, "benchmark_family_count": 8, "categories_covered": [ "agentic", "coding", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 9, "raw_model_ids": [ "openai/gpt-oss-20b" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 416, "min": 0.0, "max": 927.5760470644791, "average": 4.030370979197517 }, "reproducibility_summary": { "results_total": 416, "has_reproducibility_gap_count": 416, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 416, "total_groups": 415, "multi_source_groups": 0, "first_party_only_groups": 3, "source_type_distribution": { "first_party": 3, "third_party": 413, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 415, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "CodeForces", "Easy Problems", "GPQA", "Hard Problems", "HarmBench", "HealthBench", "HealthBench Hard", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "Medium Problems", "Omni-MATH", "SciArena leaderboard API", "SimpleSafetyTests", "Terminal Bench 2 0", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 221.225, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 89.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 24.5, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 18.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 9.442, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Terminal Bench 2 0", "benchmarkKey": "terminal_bench_2_0", "canonical_display_name": "Terminal bench 2 0 / Accuracy", "evaluation_name": "terminal-bench-2.0", "score": 3.4, "metric": "Task resolution accuracy across 87 terminal tasks with 5 trials each", "lower_is_better": false }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Self-harm / Refusal Rate", "evaluation_name": "AIRBench 2024 - Self-harm", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Self-harm", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.997, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.987, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.967, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.964, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime_25", "canonical_display_name": "artificial_analysis.aime_25 / Aime 25", "evaluation_name": "artificial_analysis.aime_25", "score": 0.893, "metric": "Benchmark score on AIME 2025.", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.872, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.777, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false } ] }, { "model_group_id": "openai/o1", "model_route_id": "openai__o1", "model_family_name": "o1", "developer": "openai", "params_billions": null, "total_evaluations": 4, "benchmark_count": 4, "benchmark_family_count": 4, "categories_covered": [ "general", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/o1" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "2024-12-17", "variant_label": "2024-12-17", "evaluation_count": 3, "raw_model_ids": [ "openai/o1-2024-12-17" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 414, "min": 0.0, "max": 105.408, "average": 1.510057971014491 }, "reproducibility_summary": { "results_total": 414, "has_reproducibility_gap_count": 414, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 414, "total_groups": 414, "multi_source_groups": 0, "first_party_only_groups": 15, "source_type_distribution": { "first_party": 15, "third_party": 399, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 414, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "FrontierMath", "GPQA Biology", "GPQA Chemistry", "GPQA Physics", "GSM8k", "HarmBench", "Helm air bench", "Helm safety", "LiveBench", "MATH", "MGSM", "MMLU", "MMMLU", "MMMU", "MathVista", "SimpleQA", "SimpleSafetyTests", "TAU-bench Airline", "TAU-bench Retail", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 105.408, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 60.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 30.8, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 26.25, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 21.793, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 21.793, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 20.5, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 15.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Self-harm / Refusal Rate", "evaluation_name": "AIRBench 2024 - Self-harm", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Self-harm", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.99, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.983, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.976, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.973, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "GSM8k", "benchmarkKey": "llm_stats_gsm8k", "canonical_display_name": "Gsm8k / Score", "evaluation_name": "llm_stats.gsm8k", "score": 0.971, "metric": "Grade School Math 8K, a dataset of 8.5K high-quality linguistically diverse grade school math word problems requiring multi-step reasoning and elementary arithmetic operations.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.97, "metric": "Benchmark score on MATH-500.", "lower_is_better": false } ] }, { "model_group_id": "openai/o1-mini", "model_route_id": "openai__o1-mini", "model_family_name": "o1-mini", "developer": "openai", "params_billions": null, "total_evaluations": 5, "benchmark_count": 5, "benchmark_family_count": 5, "categories_covered": [ "general", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2024-09-12", "variant_label": "2024-09-12", "evaluation_count": 3, "raw_model_ids": [ "openai/o1-mini-2024-09-12" ], "last_updated": "2026-04-25T00:19:43.227658Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/o1-mini" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 405, "min": 0.0, "max": 20.4, "average": 0.4697316049382714 }, "reproducibility_summary": { "results_total": 405, "has_reproducibility_gap_count": 405, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 405, "total_groups": 405, "multi_source_groups": 0, "first_party_only_groups": 3, "source_type_distribution": { "first_party": 3, "third_party": 402, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 405, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "Cybersecurity CTFs", "HarmBench", "Helm air bench", "Helm safety", "HumanEval", "SimpleSafetyTests", "SuperGLUE", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 20.4, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #39-40.23: Characterization of identity - Color / Refusal Rate", "evaluation_name": "AIRBench 2024 - #39-40.23: Characterization of identity - Color", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #39-40.23: Characterization of identity - Color", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.983, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.97, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "XSTest", "benchmarkKey": "helm_safety_xstest", "canonical_display_name": "XSTest / LM Evaluated Safety score", "evaluation_name": "XSTest", "score": 0.97, "metric": "LM Evaluated Safety score on XSTest", "lower_is_better": false }, { "benchmark": "BBQ", "benchmarkKey": "helm_safety_bbq", "canonical_display_name": "BBQ / BBQ accuracy", "evaluation_name": "BBQ", "score": 0.969, "metric": "BBQ accuracy on BBQ", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.955, "metric": "The mean of the scores from all columns.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.944, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "HumanEval", "benchmarkKey": "llm_stats_humaneval", "canonical_display_name": "Humaneval / Score", "evaluation_name": "llm_stats.humaneval", "score": 0.924, "metric": "A benchmark that measures functional correctness for synthesizing programs from docstrings, consisting of 164 original programming problems assessing language comprehension, algorithms, and simple mathematics", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.885, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false }, { "benchmark": "SuperGLUE", "benchmarkKey": "llm_stats_superglue", "canonical_display_name": "Superglue / Score", "evaluation_name": "llm_stats.superglue", "score": 0.75, "metric": "SuperGLUE is a new benchmark styled after GLUE with a new set of more difficult language understanding tasks, improved resources, and a new public leaderboard. It includes 8 primary tasks: BoolQ (Boolean Questions), CB (CommitmentBank), COPA (Choice of Plausible Alternatives), MultiRC (Multi-Sentence Reading Comprehension), ReCoRD (Reading Comprehension with Commonsense Reasoning), RTE (Recognizing Textual Entailment), WiC (Word-in-Context), and WSC (Winograd Schema Challenge). The benchmark evaluates diverse language understanding capabilities including reading comprehension, commonsense reasoning, causal reasoning, coreference resolution, textual entailment, and word sense disambiguation across multiple domains.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.742, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.603, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.603, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.576, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false } ] }, { "model_group_id": "openai/o1-preview", "model_route_id": "openai__o1-preview", "model_family_name": "o1-preview", "developer": "openai", "params_billions": null, "total_evaluations": 3, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "agentic", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 3, "raw_model_ids": [ "openai/o1-preview" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 15, "min": 0.0, "max": 66.0, "average": 11.61753333333333 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 5, "source_type_distribution": { "first_party": 5, "third_party": 10, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API", "LiveBench", "MATH", "MGSM", "MMLU", "SWE-bench Verified", "SimpleQA" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 66.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 34.0, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 28.875, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 23.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 16.5, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.924, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "MGSM", "benchmarkKey": "llm_stats_mgsm", "canonical_display_name": "Mgsm / Score", "evaluation_name": "llm_stats.mgsm", "score": 0.908, "metric": "MGSM (Multilingual Grade School Math) is a benchmark of grade-school math problems. Contains 250 grade-school math problems manually translated from the GSM8K dataset into ten typologically diverse languages: Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, and Telugu. Evaluates multilingual mathematical reasoning capabilities.", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "llm_stats_mmlu", "canonical_display_name": "Mmlu / Score", "evaluation_name": "llm_stats.mmlu", "score": 0.908, "metric": "Massive Multitask Language Understanding benchmark testing knowledge across 57 diverse subjects including STEM, humanities, social sciences, and professional domains", "lower_is_better": false }, { "benchmark": "MATH", "benchmarkKey": "llm_stats_math", "canonical_display_name": "Math / Score", "evaluation_name": "llm_stats.math", "score": 0.855, "metric": "MATH dataset contains 12,500 challenging competition mathematics problems from AMC 10, AMC 12, AIME, and other mathematics competitions. Each problem includes full step-by-step solutions and spans multiple difficulty levels (1-5) across seven mathematical subjects including Prealgebra, Algebra, Number Theory, Counting and Probability, Geometry, Intermediate Algebra, and Precalculus.", "lower_is_better": false }, { "benchmark": "SWE-bench Verified", "benchmarkKey": "swe_bench_verified", "canonical_display_name": "SWE-bench Verified / Score", "evaluation_name": "SWE-bench Verified", "score": 0.646, "metric": "Fraction of 500 verified GitHub issues resolved (0.0–1.0)", "lower_is_better": false }, { "benchmark": "LiveBench", "benchmarkKey": "llm_stats_livebench", "canonical_display_name": "Livebench / Score", "evaluation_name": "llm_stats.livebench", "score": 0.523, "metric": "LiveBench is a challenging, contamination-limited LLM benchmark that addresses test set contamination by releasing new questions monthly based on recently-released datasets, arXiv papers, news articles, and IMDb movie synopses. It comprises tasks across math, coding, reasoning, language, instruction following, and data analysis with verifiable, objective ground-truth answers.", "lower_is_better": false }, { "benchmark": "SimpleQA", "benchmarkKey": "llm_stats_simpleqa", "canonical_display_name": "Simpleqa / Score", "evaluation_name": "llm_stats.simpleqa", "score": 0.424, "metric": "SimpleQA is a factuality benchmark developed by OpenAI that measures the short-form factual accuracy of large language models. The benchmark contains 4,326 short, fact-seeking questions that are adversarially collected and designed to have single, indisputable answers. Questions cover diverse topics from science and technology to entertainment, and the benchmark also measures model calibration by evaluating whether models know what they know.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "openai/o1-pro", "model_route_id": "openai__o1-pro", "model_family_name": "o1-pro", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "other" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/o1-pro" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 8, "min": 0.0, "max": 600.0, "average": 129.89499999999998 }, "reproducibility_summary": { "results_total": 8, "has_reproducibility_gap_count": 8, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 8, "total_groups": 8, "multi_source_groups": 0, "first_party_only_groups": 1, "source_type_distribution": { "first_party": 1, "third_party": 7, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 8, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AIME 2024", "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 600.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 262.5, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 150.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 25.8, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "AIME 2024", "benchmarkKey": "llm_stats_aime_2024", "canonical_display_name": "Aime 2024 / Score", "evaluation_name": "llm_stats.aime-2024", "score": 0.86, "metric": "American Invitational Mathematics Examination 2024, consisting of 30 challenging mathematical reasoning problems from AIME I and AIME II competitions. Each problem requires an integer answer between 0-999 and tests advanced mathematical reasoning across algebra, geometry, combinatorics, and number theory. Used as a benchmark for evaluating mathematical reasoning capabilities in large language models at Olympiad-level difficulty.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 0.0, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 0.0, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 0.0, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true } ] }, { "model_group_id": "openai/o3", "model_route_id": "openai__o3", "model_family_name": "o3", "developer": "openai", "params_billions": null, "total_evaluations": 15, "benchmark_count": 11, "benchmark_family_count": 11, "categories_covered": [ "coding", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 4, "raw_model_ids": [ "openai/o3" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "2025-04-16-high", "variant_label": "2025-04-16 high", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-2025-04-16-high" ], "last_updated": "2026-04-07T08:15:57.610323Z" }, { "variant_key": "2025-04-16-low", "variant_label": "2025-04-16 low", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-2025-04-16-low" ], "last_updated": "2026-04-07T08:15:57.610641Z" }, { "variant_key": "2025-04-16-medium", "variant_label": "2025-04-16 medium", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-2025-04-16-medium" ], "last_updated": "2026-04-07T08:15:57.611015Z" }, { "variant_key": "2025-04-16", "variant_label": "2025-04-16", "evaluation_count": 8, "raw_model_ids": [ "openai/o3-2025-04-16", "openai/o3-2025-04-16-fc", "openai/o3-2025-04-16-prompt" ], "last_updated": "2026-04-25T09:07:44.422824Z" } ], "score_summary": { "count": 519, "min": 0.0, "max": 1151.3779287263492, "average": 9.743169731595739 }, "reproducibility_summary": { "results_total": 519, "has_reproducibility_gap_count": 519, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 519, "total_groups": 473, "multi_source_groups": 0, "first_party_only_groups": 19, "source_type_distribution": { "first_party": 19, "third_party": 499, "collaborative": 0, "unspecified": 1 } }, "comparability_summary": { "total_groups": 473, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ACE", "AIME 2024", "APEX v1", "ARC Prize evaluations leaderboard JSON", "ARC-AGI", "ARC-AGI v2", "Aider-Polyglot", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BFCL leaderboard CSV", "COLLIE", "CharXiv-R", "ERQA", "Easy Problems", "FrontierMath", "GPQA", "Hard Problems", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "MMMU", "MMMU-Pro", "MathVista", "Medium Problems", "Multi-Challenge", "Omni-MATH", "SciArena leaderboard API", "SimpleSafetyTests", "Tau-bench", "Tau2 Airline", "Tau2 Retail", "VideoMMMU", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 88.3, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 77.093, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 38.4, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 38.4, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 8.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 7.001, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 7.001, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.5, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 2.75, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 2.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Self-harm / Refusal Rate", "evaluation_name": "AIRBench 2024 - Self-harm", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Self-harm", "lower_is_better": false }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "canonical_display_name": "SciArena / Rank", "evaluation_name": "overall_rank", "score": 1.0, "metric": "overall_rank", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.992, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.99, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "HarmBench", "benchmarkKey": "helm_safety_harmbench", "canonical_display_name": "HarmBench / LM Evaluated Safety score", "evaluation_name": "HarmBench", "score": 0.984, "metric": "LM Evaluated Safety score on HarmBench", "lower_is_better": false } ] }, { "model_group_id": "openai/o3-mini", "model_route_id": "openai__o3-mini", "model_family_name": "o3-mini", "developer": "openai", "params_billions": null, "total_evaluations": 10, "benchmark_count": 7, "benchmark_family_count": 7, "categories_covered": [ "agentic", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-01-31-high", "variant_label": "2025-01-31 high", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-mini-2025-01-31-high" ], "last_updated": "2026-04-07T08:15:57.611337Z" }, { "variant_key": "2025-01-31-low", "variant_label": "2025-01-31 low", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-mini-2025-01-31-low" ], "last_updated": "2026-04-07T08:15:57.611654Z" }, { "variant_key": "2025-01-31-medium", "variant_label": "2025-01-31 medium", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-mini-2025-01-31-medium" ], "last_updated": "2026-04-07T08:15:57.611961Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 3, "raw_model_ids": [ "openai/o3-mini" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "2025-01-31", "variant_label": "2025-01-31", "evaluation_count": 4, "raw_model_ids": [ "openai/o3-mini-2025-01-31" ], "last_updated": "2026-04-25T00:19:43.227658Z" } ], "score_summary": { "count": 479, "min": 0.0, "max": 150.312, "average": 1.1567396659707732 }, "reproducibility_summary": { "results_total": 479, "has_reproducibility_gap_count": 479, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 479, "total_groups": 444, "multi_source_groups": 0, "first_party_only_groups": 19, "source_type_distribution": { "first_party": 19, "third_party": 460, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 444, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AIME 2024", "ARC Prize evaluations leaderboard JSON", "Aider-Polyglot", "Aider-Polyglot Edit", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "COLLIE", "ComplexFuncBench", "FrontierMath", "Global-MMLU Lite", "Graphwalks BFS <128k", "Graphwalks parents <128k", "HarmBench", "Helm air bench", "Helm safety", "IFEval", "Internal API instruction following (hard)", "LiveBench", "MATH", "MGSM", "Multi-Challenge", "Multi-IF", "Multilingual MMLU", "OpenAI-MRCR: 2 needle 128k", "SWE-Lancer", "SWE-Lancer (IC-Diamond subset)", "SWE-bench Verified", "SimpleSafetyTests", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 150.312, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 25.9, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 17.9, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 6.674, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 6.674, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 4.4, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 1.925, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.1, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - #19: Non-Consensual Nudity / Refusal Rate", "evaluation_name": "AIRBench 2024 - #19: Non-Consensual Nudity", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - #19: Non-Consensual Nudity", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 0.99, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "COLLIE", "benchmarkKey": "llm_stats_collie", "canonical_display_name": "Collie / Score", "evaluation_name": "llm_stats.collie", "score": 0.987, "metric": "COLLIE is a grammar-based framework for systematic construction of constrained text generation tasks. It allows specification of rich, compositional constraints across diverse generation levels and modeling challenges including language understanding, logical reasoning, and semantic planning. The COLLIE-v1 dataset contains 2,080 instances across 13 constraint structures.", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.986, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false }, { "benchmark": "MATH", "benchmarkKey": "llm_stats_math", "canonical_display_name": "Math / Score", "evaluation_name": "llm_stats.math", "score": 0.979, "metric": "MATH dataset contains 12,500 challenging competition mathematics problems from AMC 10, AMC 12, AIME, and other mathematics competitions. Each problem includes full step-by-step solutions and spans multiple difficulty levels (1-5) across seven mathematical subjects including Prealgebra, Algebra, Number Theory, Counting and Probability, Geometry, Intermediate Algebra, and Precalculus.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.973, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Helm safety", "benchmarkKey": "helm_safety", "canonical_display_name": "Safety / Mean / Score", "evaluation_name": "Mean score", "score": 0.962, "metric": "The mean of the scores from all columns.", "lower_is_better": false } ] }, { "model_group_id": "openai/o3-mini-high", "model_route_id": "openai__o3-mini-high", "model_family_name": "o3-mini (high)", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "other" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-mini-high" ], "last_updated": "2026-04-11T14:48:41.622802Z" } ], "score_summary": { "count": 19, "min": 0.061, "max": 143.11, "average": 12.978736842105263 }, "reproducibility_summary": { "results_total": 19, "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 19, "total_groups": 19, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 19, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 19, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 143.11, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 25.2, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 23.724, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 23.724, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 17.3, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 4.4, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 1.925, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.1, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.985, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_aime", "canonical_display_name": "artificial_analysis.aime / AIME", "evaluation_name": "artificial_analysis.aime", "score": 0.86, "metric": "Benchmark score on AIME.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "canonical_display_name": "artificial_analysis.mmlu_pro / MMLU-Pro", "evaluation_name": "artificial_analysis.mmlu_pro", "score": 0.802, "metric": "Benchmark score on MMLU-Pro.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.773, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_livecodebench", "canonical_display_name": "artificial_analysis.livecodebench / LiveCodeBench", "evaluation_name": "artificial_analysis.livecodebench", "score": 0.734, "metric": "Benchmark score on LiveCodeBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_ifbench", "canonical_display_name": "artificial_analysis.ifbench / IFBench", "evaluation_name": "artificial_analysis.ifbench", "score": 0.671, "metric": "Benchmark score on IFBench.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_scicode", "canonical_display_name": "artificial_analysis.scicode / SciCode", "evaluation_name": "artificial_analysis.scicode", "score": 0.398, "metric": "Benchmark score on SciCode.", "lower_is_better": false } ] }, { "model_group_id": "openai/o3-pro", "model_route_id": "openai__o3-pro", "model_family_name": "o3-pro", "developer": "openai", "params_billions": null, "total_evaluations": 5, "benchmark_count": 3, "benchmark_family_count": 3, "categories_covered": [ "other", "reasoning" ], "last_updated": "2026-04-11T14:48:41.622802Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/o3 Pro", "openai/o3-pro" ], "last_updated": "2026-04-11T14:48:41.622802Z" }, { "variant_key": "2025-06-10-high", "variant_label": "2025-06-10 high", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-pro-2025-06-10-high" ], "last_updated": "2026-04-07T08:15:57.612277Z" }, { "variant_key": "2025-06-10-low", "variant_label": "2025-06-10 low", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-pro-2025-06-10-low" ], "last_updated": "2026-04-07T08:15:57.612581Z" }, { "variant_key": "2025-06-10-medium", "variant_label": "2025-06-10 medium", "evaluation_count": 1, "raw_model_ids": [ "openai/o3-pro-2025-06-10-medium" ], "last_updated": "2026-04-07T08:15:57.612879Z" } ], "score_summary": { "count": 37, "min": 0.0194, "max": 94.649, "average": 11.933337837837838 }, "reproducibility_summary": { "results_total": 37, "has_reproducibility_gap_count": 37, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 37, "total_groups": 21, "multi_source_groups": 0, "first_party_only_groups": 5, "source_type_distribution": { "first_party": 5, "third_party": 32, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 21, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "ACE", "ARC Prize evaluations leaderboard JSON", "Artificial Analysis LLM API" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 94.649, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 94.649, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 80.0, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 40.7, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 35.0, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 21.178, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 20.0, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_gpqa", "canonical_display_name": "artificial_analysis.gpqa / GPQA", "evaluation_name": "artificial_analysis.gpqa", "score": 0.845, "metric": "Benchmark score on GPQA.", "lower_is_better": false }, { "benchmark": "ARC Prize evaluations leaderboard JSON", "benchmarkKey": "arc_agi", "canonical_display_name": "ARC-AGI / v1_Public_Eval / Score", "evaluation_name": "v1_Public_Eval", "score": 0.6334, "metric": "v1_Public_Eval", "lower_is_better": false }, { "benchmark": "ACE", "benchmarkKey": "ace", "canonical_display_name": "ACE / Gaming / Score", "evaluation_name": "Gaming", "score": 0.613, "metric": "Gaming domain score.", "lower_is_better": false } ] }, { "model_group_id": "openai/o4-mini", "model_route_id": "openai__o4-mini", "model_family_name": "o4-mini", "developer": "openai", "params_billions": null, "total_evaluations": 15, "benchmark_count": 11, "benchmark_family_count": 11, "categories_covered": [ "agentic", "coding", "general", "knowledge", "other", "reasoning" ], "last_updated": "2026-04-25T09:07:44.422824Z", "variants": [ { "variant_key": "2025-04-16-high", "variant_label": "2025-04-16 high", "evaluation_count": 1, "raw_model_ids": [ "openai/o4-mini-2025-04-16-high" ], "last_updated": "2026-04-07T08:15:57.613200Z" }, { "variant_key": "2025-04-16-low", "variant_label": "2025-04-16 low", "evaluation_count": 1, "raw_model_ids": [ "openai/o4-mini-2025-04-16-low" ], "last_updated": "2026-04-07T08:15:57.613509Z" }, { "variant_key": "2025-04-16-medium", "variant_label": "2025-04-16 medium", "evaluation_count": 1, "raw_model_ids": [ "openai/o4-mini-2025-04-16-medium" ], "last_updated": "2026-04-07T08:15:57.613811Z" }, { "variant_key": "default", "variant_label": "Default", "evaluation_count": 4, "raw_model_ids": [ "openai/o4-mini" ], "last_updated": "2026-04-25T09:07:44.422824Z" }, { "variant_key": "2025-04-16", "variant_label": "2025-04-16", "evaluation_count": 8, "raw_model_ids": [ "openai/o4-mini-2025-04-16", "openai/o4-mini-2025-04-16-fc", "openai/o4-mini-2025-04-16-prompt" ], "last_updated": "2026-04-25T00:19:43.227658Z" } ], "score_summary": { "count": 531, "min": 0.0, "max": 1028.5553925129395, "average": 8.773015932426702 }, "reproducibility_summary": { "results_total": 531, "has_reproducibility_gap_count": 531, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 531, "total_groups": 482, "multi_source_groups": 0, "first_party_only_groups": 9, "source_type_distribution": { "first_party": 9, "third_party": 522, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 482, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "AIME 2024", "ARC Prize evaluations leaderboard JSON", "Aider-Polyglot", "Aider-Polyglot Edit", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BFCL leaderboard CSV", "CharXiv-R", "Easy Problems", "GPQA", "Global-MMLU Lite", "Hard Problems", "HarmBench", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "IFEval", "MMLU-Pro", "MMMU", "MathVista", "Medium Problems", "Multi-Challenge", "Omni-MATH", "SWE-bench Verified", "SciArena leaderboard API", "SimpleSafetyTests", "TAU-bench Airline", "TAU-bench Retail", "WildBench", "XSTest" ], "top_benchmark_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "canonical_display_name": "artificial_analysis.median_output_tokens_per_second / Median output tokens per second", "evaluation_name": "artificial_analysis.median_output_tokens_per_second", "score": 134.085, "metric": "Median output generation speed reported by Artificial Analysis.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "canonical_display_name": "artificial_analysis.artificial_analysis_math_index / Artificial Analysis Math Index", "evaluation_name": "artificial_analysis.artificial_analysis_math_index", "score": 90.7, "metric": "Artificial Analysis composite math index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "canonical_display_name": "artificial_analysis.artificial_analysis_intelligence_index / Artificial Analysis Intelligence Index", "evaluation_name": "artificial_analysis.artificial_analysis_intelligence_index", "score": 33.1, "metric": "Artificial Analysis composite intelligence index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "canonical_display_name": "artificial_analysis.median_time_to_first_token_seconds / Median Time To First Token Seconds", "evaluation_name": "artificial_analysis.median_time_to_first_token_seconds", "score": 28.202, "metric": "Median time to first token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "canonical_display_name": "artificial_analysis.median_time_to_first_answer_token / Median time to first answer token", "evaluation_name": "artificial_analysis.median_time_to_first_answer_token", "score": 28.202, "metric": "Median time to first answer token reported by Artificial Analysis.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "canonical_display_name": "artificial_analysis.artificial_analysis_coding_index / Artificial Analysis Coding Index", "evaluation_name": "artificial_analysis.artificial_analysis_coding_index", "score": 25.6, "metric": "Artificial Analysis composite coding index.", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "canonical_display_name": "artificial_analysis.price_1m_output_tokens / Price 1m Output Tokens", "evaluation_name": "artificial_analysis.price_1m_output_tokens", "score": 4.4, "metric": "Price per 1M output tokens in USD.", "lower_is_better": true }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "canonical_display_name": "Bfcl / Format sensitivity / Format Sensitivity Standard Deviation", "evaluation_name": "format_sensitivity", "score": 2.6, "metric": "format_sensitivity", "lower_is_better": true }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "canonical_display_name": "SciArena / Cost per 100 Calls", "evaluation_name": "overall_cost_per_100_calls_usd", "score": 2.4773, "metric": "overall_cost_per_100_calls_usd", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "canonical_display_name": "artificial_analysis.price_1m_blended_3_to_1 / Price 1m Blended 3 To 1", "evaluation_name": "artificial_analysis.price_1m_blended_3_to_1", "score": 1.925, "metric": "Blended price per 1M tokens using a 3:1 input-to-output ratio.", "lower_is_better": true }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "canonical_display_name": "artificial_analysis.price_1m_input_tokens / Price 1m Input Tokens", "evaluation_name": "artificial_analysis.price_1m_input_tokens", "score": 1.1, "metric": "Price per 1M input tokens in USD.", "lower_is_better": true }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "canonical_display_name": "Air bench / AIRBench 2024 - Self-harm / Refusal Rate", "evaluation_name": "AIRBench 2024 - Self-harm", "score": 1.0, "metric": "Refusal Rate on AIRBench 2024 - Self-harm", "lower_is_better": false }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evaluation_name": "SimpleSafetyTests", "score": 1.0, "metric": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "canonical_display_name": "artificial_analysis.math_500 / MATH-500", "evaluation_name": "artificial_analysis.math_500", "score": 0.989, "metric": "Benchmark score on MATH-500.", "lower_is_better": false }, { "benchmark": "Anthropic Red Team", "benchmarkKey": "helm_safety_anthropic_red_team", "canonical_display_name": "Anthropic Red Team / LM Evaluated Safety score", "evaluation_name": "Anthropic Red Team", "score": 0.982, "metric": "LM Evaluated Safety score on Anthropic Red Team", "lower_is_better": false } ] }, { "model_group_id": "openai/gpt-4-0125-preview", "model_route_id": "openai__gpt-4-0125-preview", "model_family_name": "openai/gpt-4-0125-preview", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "reasoning" ], "last_updated": "2025-12-22T14:13:58.146816Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/gpt-4-0125-preview" ], "last_updated": "2025-12-22T14:13:58.146816Z" } ], "score_summary": { "count": 6, "min": 0.7085, "max": 0.9525, "average": 0.8321166666666667 }, "reproducibility_summary": { "results_total": 6, "has_reproducibility_gap_count": 6, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 6, "total_groups": 6, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 6, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 6, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "RewardBench" ], "top_benchmark_scores": [ { "benchmark": "RewardBench", "benchmarkKey": "reward_bench", "canonical_display_name": "Reward Bench / Chat / Accuracy", "evaluation_name": "Chat", "score": 0.9525, "metric": "Chat accuracy - includes easy chat subsets", "lower_is_better": false }, { "benchmark": "RewardBench", "benchmarkKey": "reward_bench_rewardbench", "canonical_display_name": "RewardBench / Score", "evaluation_name": "RewardBench", "score": 0.8434, "metric": "Overall RewardBench Score", "lower_is_better": false } ] }, { "model_group_id": "openai/text-ada-001", "model_route_id": "openai__text-ada-001", "model_family_name": "text-ada-001", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/text-ada-001" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 15, "min": 0.034, "max": 0.822, "average": 0.30546666666666666 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.822, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.503, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.464, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.429, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.406, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.346, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.302, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.238, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.238, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.232, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "QuAC", "benchmarkKey": "helm_classic_quac", "canonical_display_name": "QuAC / F1", "evaluation_name": "QuAC", "score": 0.176, "metric": "F1 on QuAC", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.149, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": 0.136, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.107, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": 0.034, "metric": "ROUGE-2 on XSUM", "lower_is_better": false } ] }, { "model_group_id": "openai/text-babbage-001", "model_route_id": "openai__text-babbage-001", "model_family_name": "text-babbage-001", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/text-babbage-001" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 15, "min": 0.046, "max": 0.913, "average": 0.3843333333333333 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.913, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.561, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.509, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.499, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.452, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.451, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.449, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.429, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.33, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "QuAC", "benchmarkKey": "helm_classic_quac", "canonical_display_name": "QuAC / F1", "evaluation_name": "QuAC", "score": 0.284, "metric": "F1 on QuAC", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.233, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.229, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.229, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": 0.151, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": 0.046, "metric": "ROUGE-2 on XSUM", "lower_is_better": false } ] }, { "model_group_id": "openai/text-curie-001", "model_route_id": "openai__text-curie-001", "model_family_name": "text-curie-001", "developer": "openai", "params_billions": null, "total_evaluations": 1, "benchmark_count": 1, "benchmark_family_count": 1, "categories_covered": [ "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 1, "raw_model_ids": [ "openai/text-curie-001" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 15, "min": 0.076, "max": 0.923, "average": 0.4572666666666666 }, "reproducibility_summary": { "results_total": 15, "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 15, "total_groups": 15, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 15, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 15, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "Helm classic", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.923, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.676, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.62, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.582, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.571, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.537, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.514, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.507, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.489, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.36, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "QuAC", "benchmarkKey": "helm_classic_quac", "canonical_display_name": "QuAC / F1", "evaluation_name": "QuAC", "score": 0.358, "metric": "F1 on QuAC", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.257, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.237, "metric": "EM on MMLU", "lower_is_better": false }, { "benchmark": "CNN/DailyMail", "benchmarkKey": "helm_classic_cnn_dailymail", "canonical_display_name": "CNN/DailyMail / ROUGE-2", "evaluation_name": "CNN/DailyMail", "score": 0.152, "metric": "ROUGE-2 on CNN/DailyMail", "lower_is_better": false }, { "benchmark": "XSUM", "benchmarkKey": "helm_classic_xsum", "canonical_display_name": "XSUM / ROUGE-2", "evaluation_name": "XSUM", "score": 0.076, "metric": "ROUGE-2 on XSUM", "lower_is_better": false } ] }, { "model_group_id": "openai/text-davinci-002", "model_route_id": "openai__text-davinci-002", "model_family_name": "text-davinci-002", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "general", "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/text-davinci-002" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 25, "min": 0.144, "max": 0.948, "average": 0.5825199999999999 }, "reproducibility_summary": { "results_total": 25, "has_reproducibility_gap_count": 25, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 25, "total_groups": 25, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 25, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 25, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "GSM8K", "HellaSwag", "Helm classic", "Helm lite", "IMDB", "LegalBench", "MATH", "MMLU", "MS MARCO (TREC)", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "WMT 2014", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.948, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.905, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.877, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.815, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.796, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.733, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.727, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.719, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.713, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.668, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.664, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.61, "metric": "EM on TruthfulQA", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.594, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "LegalBench", "benchmarkKey": "helm_lite_legalbench", "canonical_display_name": "LegalBench / Exact Match", "evaluation_name": "LegalBench", "score": 0.58, "metric": "EM on LegalBench", "lower_is_better": false }, { "benchmark": "MMLU", "benchmarkKey": "helm_classic_mmlu", "canonical_display_name": "MMLU / Exact Match", "evaluation_name": "MMLU", "score": 0.568, "metric": "EM on MMLU", "lower_is_better": false } ] }, { "model_group_id": "openai/text-davinci-003", "model_route_id": "openai__text-davinci-003", "model_family_name": "text-davinci-003", "developer": "openai", "params_billions": null, "total_evaluations": 2, "benchmark_count": 2, "benchmark_family_count": 2, "categories_covered": [ "general", "knowledge", "language_understanding", "reasoning", "safety" ], "last_updated": "2026-03-21T12:31:48.339228Z", "variants": [ { "variant_key": "default", "variant_label": "Default", "evaluation_count": 2, "raw_model_ids": [ "openai/text-davinci-003" ], "last_updated": "2026-03-21T12:31:48.339228Z" } ], "score_summary": { "count": 25, "min": 0.124, "max": 0.881, "average": 0.59976 }, "reproducibility_summary": { "results_total": 25, "has_reproducibility_gap_count": 25, "populated_ratio_avg": 0.0 }, "provenance_summary": { "total_results": 25, "total_groups": 25, "multi_source_groups": 0, "first_party_only_groups": 0, "source_type_distribution": { "first_party": 0, "third_party": 25, "collaborative": 0, "unspecified": 0 } }, "comparability_summary": { "total_groups": 25, "groups_with_variant_check": 0, "groups_with_cross_party_check": 0, "variant_divergent_count": 0, "cross_party_divergent_count": 0 }, "benchmark_names": [ "BoolQ", "CNN/DailyMail", "CivilComments", "GSM8K", "HellaSwag", "Helm classic", "Helm lite", "IMDB", "LegalBench", "MATH", "MMLU", "MS MARCO (TREC)", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "WMT 2014", "XSUM" ], "top_benchmark_scores": [ { "benchmark": "BoolQ", "benchmarkKey": "helm_classic_boolq", "canonical_display_name": "BoolQ / Exact Match", "evaluation_name": "BoolQ", "score": 0.881, "metric": "EM on BoolQ", "lower_is_better": false }, { "benchmark": "Helm classic", "benchmarkKey": "helm_classic", "canonical_display_name": "Helm classic / Win Rate", "evaluation_name": "helm_classic", "score": 0.872, "metric": "How many models this model outperform on average (over columns).", "lower_is_better": false }, { "benchmark": "IMDB", "benchmarkKey": "helm_classic_imdb", "canonical_display_name": "IMDB / Exact Match", "evaluation_name": "IMDB", "score": 0.848, "metric": "EM on IMDB", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.828, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "HellaSwag", "benchmarkKey": "helm_classic_hellaswag", "canonical_display_name": "HellaSwag / Exact Match", "evaluation_name": "HellaSwag", "score": 0.822, "metric": "EM on HellaSwag", "lower_is_better": false }, { "benchmark": "NaturalQuestions (open-book)", "benchmarkKey": "helm_classic_naturalquestions_open_book", "canonical_display_name": "NaturalQuestions (open-book) / F1", "evaluation_name": "NaturalQuestions (open-book)", "score": 0.77, "metric": "F1 on NaturalQuestions (open-book)", "lower_is_better": false }, { "benchmark": "RAFT", "benchmarkKey": "helm_classic_raft", "canonical_display_name": "RAFT / Exact Match", "evaluation_name": "RAFT", "score": 0.759, "metric": "EM on RAFT", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.731, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_classic_narrativeqa", "canonical_display_name": "NarrativeQA / F1", "evaluation_name": "NarrativeQA", "score": 0.727, "metric": "F1 on NarrativeQA", "lower_is_better": false }, { "benchmark": "CivilComments", "benchmarkKey": "helm_classic_civilcomments", "canonical_display_name": "CivilComments / Exact Match", "evaluation_name": "CivilComments", "score": 0.684, "metric": "EM on CivilComments", "lower_is_better": false }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_classic_openbookqa", "canonical_display_name": "OpenbookQA / Exact Match", "evaluation_name": "OpenbookQA", "score": 0.646, "metric": "EM on OpenbookQA", "lower_is_better": false }, { "benchmark": "MS MARCO (TREC)", "benchmarkKey": "helm_classic_ms_marco_trec", "canonical_display_name": "MS MARCO (TREC) / NDCG", "evaluation_name": "MS MARCO (TREC)", "score": 0.644, "metric": "NDCG@10 on MS MARCO (TREC)", "lower_is_better": false }, { "benchmark": "LegalBench", "benchmarkKey": "helm_lite_legalbench", "canonical_display_name": "LegalBench / Exact Match", "evaluation_name": "LegalBench", "score": 0.622, "metric": "EM on LegalBench", "lower_is_better": false }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "canonical_display_name": "GSM8K / Exact Match", "evaluation_name": "GSM8K", "score": 0.615, "metric": "EM on GSM8K", "lower_is_better": false }, { "benchmark": "TruthfulQA", "benchmarkKey": "helm_classic_truthfulqa", "canonical_display_name": "TruthfulQA / Exact Match", "evaluation_name": "TruthfulQA", "score": 0.593, "metric": "EM on TruthfulQA", "lower_is_better": false } ] } ] }