// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`flattenModelEvaluations > ai21__j1-grande-v1-17b — Has \`safety\` hierarchy_by_category key 1`] = ` { "count": 15, "distinct_categories": [ "General", "Knowledge", "Reasoning", "Safety", ], "distinct_evaluation_ids": 15, "distinct_evaluator_relationships": [ "third_party", ], "distinct_family_ids": 1, "full_output_hash": "da9bc4816d9c", "missing_source_metadata": 0, } `; exports[`flattenModelEvaluations > bytedance__seed-2-0-lite — Has \`coding\` hierarchy_by_category key (the substring-fallacy bug case); small 1`] = ` { "count": 2, "distinct_categories": [ "General", ], "distinct_evaluation_ids": 2, "distinct_evaluator_relationships": [ "first_party", ], "distinct_family_ids": 1, "full_output_hash": "d6ec47a8cac2", "missing_source_metadata": 0, } `; exports[`flattenModelEvaluations > google__gemini-3-flash — Already covered by parity harness; medium-size; multi-category hierarchy 1`] = ` { "count": 29, "distinct_categories": [ "Agentic", "General", "Knowledge", ], "distinct_evaluation_ids": 29, "distinct_evaluator_relationships": [ "first_party", "third_party", ], "distinct_family_ids": 5, "full_output_hash": "5a459c11c678", "missing_source_metadata": 0, } `; exports[`flattenModelEvaluations > openai__gpt-5-2-pro — 5 variants — exercises variant_lookup and per-variant grouping in flattenModelEvaluations 1`] = ` { "count": 30, "distinct_categories": [ "General", "Knowledge", "Reasoning", ], "distinct_evaluation_ids": 30, "distinct_evaluator_relationships": [ "first_party", "third_party", ], "distinct_family_ids": 3, "full_output_hash": "debca90cf490", "missing_source_metadata": 0, } `; exports[`hfDeveloperDetailToSummary > 01-ai — Dash-prefix slug — exercises pipelineSlugify edge case 1`] = ` { "benchmark_count": 17, "developer": "01-ai", "evaluation_count": 26, "model_count": 20, "models_count": 20, "models_hash": "8790db11b51f", "popular_evals": [ { "benchmark": "BBH", "model_count": 19, }, { "benchmark": "GPQA", "model_count": 19, }, { "benchmark": "IFEval", "model_count": 19, }, ], "route_id": "01-ai", } `; exports[`hfDeveloperDetailToSummary > anthropic — Multiple model families; typical case 1`] = ` { "benchmark_count": 110, "developer": "Anthropic", "evaluation_count": 333, "model_count": 58, "models_count": 58, "models_hash": "c5bb564f52a3", "popular_evals": [ { "benchmark": "Artificial Analysis LLM API", "model_count": 28, }, { "benchmark": "ARC Prize evaluations leaderboard JSON", "model_count": 16, }, { "benchmark": "GPQA", "model_count": 11, }, ], "route_id": "anthropic", } `; exports[`hfDeveloperDetailToSummary > openai — KNOWN_DEVELOPER_NAMES canonicalization (openai → OpenAI) 1`] = ` { "benchmark_count": 163, "developer": "OpenAI", "evaluation_count": 490, "model_count": 111, "models_count": 111, "models_hash": "e8d6951b2ce7", "popular_evals": [ { "benchmark": "Artificial Analysis LLM API", "model_count": 57, }, { "benchmark": "ARC Prize evaluations leaderboard JSON", "model_count": 33, }, { "benchmark": "MMLU", "model_count": 25, }, ], "route_id": "openai", } `; exports[`hfEvalDetailToSummary > apex_v1 — Mercor first-party badge; pipeline category=knowledge; subtasks 1`] = ` { "avg_score": 0.6027142857142858, "avg_score_norm": 0.6027142857142858, "benchmark_card": { "benchmark_details": { "benchmark_type": "single", "data_type": "text", "domains": [ "investment banking", "management consulting", "corporate law", "finance", "legal", "consulting", ], "languages": [ "English", ], "name": "APEX-v1", "overview": "APEX-Agents (AI Productivity Index for Agents) measures the ability of AI agents to execute long-horizon, cross-application tasks created by investment banking analysts, management consultants, and corporate lawyers. The benchmark contains 480 tasks and requires agents to navigate realistic work environments with files and tools.", "resources": [ "https://arxiv.org/abs/2601.14242", "https://huggingface.co/datasets/Mercor/APEX-v1", ], "similar_benchmarks": [ "Not specified", ], }, "card_info": { "created_at": "2026-04-14T14:28:12.501639", "llm": "deepseek-ai/DeepSeek-V3.1", }, "data": { "annotation": "Tasks were created by professionals using files from within each project environment. A baselining study was conducted where independent experts executed 20% of tasks (96 tasks) to verify task feasibility, rubric fairness, and time estimates.", "format": "The specific structure of individual data instances is not described", "size": "480 tasks", "source": "The benchmark data was created by industry professionals including investment banking analysts, management consultants, and corporate lawyers. These professionals were organized into teams, assigned specific roles, and tasked with delivering complete projects over 5-10 day periods, producing high-quality customer-ready deliverables from scratch.", }, "ethical_and_legal_considerations": { "compliance_with_regulations": "Not specified", "consent_procedures": "Not specified", "data_licensing": "Creative Commons Attribution 4.0", "privacy_and_anonymity": "Not specified", }, "flagged_fields": {}, "methodology": { "baseline_results": "Gemini 3 Flash (Thinking=High): 24.0%, GPT-5.2 (Thinking=High): 23.0%, Claude Opus 4.5 (Thinking=High): [score not specified], Gemini 3 Pro (Thinking=High): [score not specified], GPT-OSS-120B (High): 15.2%, Grok 4: 0%", "calculation": "The overall Pass@1 score is computed as the task-uniform mean of per-task pass rates across all 480 tasks. Confidence intervals are calculated using task-level bootstrapping with 10,000 resamples", "interpretation": "Higher Pass@1 scores indicate better performance", "methods": [ "Models are evaluated using agent execution in realistic environments", "Eight trajectories are collected for each agent-task pair, with each trajectory scored as pass or fail", ], "metrics": [ "Pass@1 (task-uniform mean of per-task pass rates)", "Pass@8 (passing at least once in eight attempts)", "Pass^8 (passing consistently on all eight attempts)", ], "validation": "Automated evaluation used a judge model with 98.5% accuracy against human-labeled ground truth. A baselining study with experts validated task feasibility and rubric fairness", }, "missing_fields": [ "benchmark_details.similar_benchmarks", "purpose_and_intended_users.out_of_scope_uses", "ethical_and_legal_considerations.privacy_and_anonymity", "ethical_and_legal_considerations.consent_procedures", "ethical_and_legal_considerations.compliance_with_regulations", ], "possible_risks": [ { "category": "Over- or under-reliance", "description": [ "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html", }, { "category": "Unrepresentative data", "description": [ "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html", }, { "category": "Incomplete AI agent evaluation", "description": [ "Evaluating the performance or accuracy or an agent is difficult because of system complexity and open-endedness.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/incomplete-ai-agent-evaluation-agentic.html", }, { "category": "Reproducibility", "description": [ "Replicating agent behavior or output can be impacted by changes or updates made to external services and tools. This impact is increased if the agent is built with generative AI.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/reproducibility-agentic.html", }, { "category": "Improper usage", "description": [ "Improper usage occurs when a model is used for a purpose that it was not originally designed for.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html", }, ], "purpose_and_intended_users": { "audience": [ "AI researchers", "Developers working on agentic systems", ], "goal": "To assess whether AI agents can reliably execute highly complex professional services work, bridging the gap between existing agentic evaluations and real-world professional workflows.", "limitations": "Differences in benchmark scores below 1 percentage point should be interpreted cautiously due to a small error rate in the automated grading system (1.9% false negative rate and 1.3% false positive rate).", "out_of_scope_uses": [ "Not specified", ], "tasks": [ "Text generation", "Question answering", "Reasoning", "Demonstrating advanced knowledge", "Using multiple applications", "Planning over long horizons within realistic project scenarios", ], }, }, "best_model": { "name": "GPT 5", "score": 0.67, }, "canonical_display_name": "APEX v1", "category": "General", "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 0, "total_groups": 19, "variant_divergent_count": 0, }, "composite_benchmark_key": "APEX v1", "composite_benchmark_name": "APEX v1", "evalcards": { "annotations": { "benchmark_comparability": { "cross_party_divergence_groups": [], "variant_divergence_groups": [], }, "reporting_completeness": { "completeness_score": 0.9285714285714286, "field_scores": [ { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.name", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.overview", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.data_type", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.domains", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.languages", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.similar_benchmarks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.resources", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.goal", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.audience", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.tasks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.limitations", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.methods", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.metrics", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.calculation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.interpretation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.baseline_results", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.validation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "score": 1, }, { "coverage_type": "partial", "field_path": "autobenchmarkcard.data", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_type", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_organization_name", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.evaluator_relationship", "score": 1, }, { "coverage_type": "reserved", "field_path": "evalcards.lifecycle_status", "score": 0, }, { "coverage_type": "reserved", "field_path": "evalcards.preregistration_url", "score": 0, }, ], "missing_required_fields": [ "evalcards.lifecycle_status", "evalcards.preregistration_url", ], "partial_fields": [], "signal_version": "1.0", "total_fields_evaluated": 28, }, }, }, "evaluation_id": "apex_v1", "evaluation_name": "APEX v1", "evaluator_names": [], "latest_source_name": "APEX v1", "leaderboard_metrics": [ { "canonical_display_name": "APEX v1 / Score", "column_key": "root:apex_v1_score", "display_name": "APEX v1 / Score", "lower_is_better": false, "metric_name": "Score", "metric_summary_id": "apex_v1_score", "scope": "root", "subtask_key": undefined, "subtask_name": undefined, "unit": "proportion", }, { "canonical_display_name": "APEX v1 / Big Law / Score", "column_key": "subtask:big_law:apex_v1_big_law_score", "display_name": "APEX v1 / Big Law / Score", "lower_is_better": false, "metric_name": "Score", "metric_summary_id": "apex_v1_big_law_score", "scope": "subtask", "subtask_key": "big_law", "subtask_name": "Big Law", "unit": "proportion", }, { "canonical_display_name": "APEX v1 / Consulting / Score", "column_key": "subtask:consulting:apex_v1_consulting_score", "display_name": "APEX v1 / Consulting / Score", "lower_is_better": false, "metric_name": "Score", "metric_summary_id": "apex_v1_consulting_score", "scope": "subtask", "subtask_key": "consulting", "subtask_name": "Consulting", "unit": "proportion", }, { "canonical_display_name": "APEX v1 / Investment Banking / Score", "column_key": "subtask:investment_banking:apex_v1_investment_banking_score", "display_name": "APEX v1 / Investment Banking / Score", "lower_is_better": false, "metric_name": "Score", "metric_summary_id": "apex_v1_investment_banking_score", "scope": "subtask", "subtask_key": "investment_banking", "subtask_name": "Investment Banking", "unit": "proportion", }, { "canonical_display_name": "APEX v1 / Medicine (MD) / Score", "column_key": "subtask:medicine_md:apex_v1_medicine_md_score", "display_name": "APEX v1 / Medicine (MD) / Score", "lower_is_better": false, "metric_name": "Score", "metric_summary_id": "apex_v1_medicine_md_score", "scope": "subtask", "subtask_key": "medicine_md", "subtask_name": "Medicine (MD)", "unit": "proportion", }, ], "leaderboard_rows": [ { "annotations_by_metric": { "root:apex_v1_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, "subtask:big_law:apex_v1_big_law_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, "subtask:investment_banking:apex_v1_investment_banking_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, "subtask:medicine_md:apex_v1_medicine_md_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 4, "model_info": { "developer": "openai", "id": "openai/gpt-5", "name": "GPT 5", }, "model_route_id": "openai__gpt-5", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "root:apex_v1_score": 0.67, "subtask:big_law:apex_v1_big_law_score": 0.78, "subtask:investment_banking:apex_v1_investment_banking_score": 0.61, "subtask:medicine_md:apex_v1_medicine_md_score": 0.66, }, }, { "annotations_by_metric": { "root:apex_v1_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, "subtask:consulting:apex_v1_consulting_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, "subtask:investment_banking:apex_v1_investment_banking_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, "subtask:medicine_md:apex_v1_medicine_md_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 4, "model_info": { "developer": "openai", "id": "openai/gpt-5-2-pro", "name": "GPT 5.2 Pro", }, "model_route_id": "openai__gpt-5-2-pro", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "root:apex_v1_score": 0.668, "subtask:consulting:apex_v1_consulting_score": 0.64, "subtask:investment_banking:apex_v1_investment_banking_score": 0.64, "subtask:medicine_md:apex_v1_medicine_md_score": 0.65, }, }, { "annotations_by_metric": { "root:apex_v1_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, "subtask:consulting:apex_v1_consulting_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, "subtask:investment_banking:apex_v1_investment_banking_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 3, "model_info": { "developer": "google", "id": "google/gemini-3-pro", "name": "Gemini 3 Pro", }, "model_route_id": "google__gemini-3-pro", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "root:apex_v1_score": 0.643, "subtask:consulting:apex_v1_consulting_score": 0.64, "subtask:investment_banking:apex_v1_investment_banking_score": 0.63, }, }, { "annotations_by_metric": { "root:apex_v1_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, "subtask:consulting:apex_v1_consulting_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 2, "model_info": { "developer": "google", "id": "google/gemini-3-flash", "name": "Gemini 3 Flash", }, "model_route_id": "google__gemini-3-flash", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "root:apex_v1_score": 0.64, "subtask:consulting:apex_v1_consulting_score": 0.64, }, }, { "annotations_by_metric": { "root:apex_v1_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-4", "name": "Grok 4", }, "model_route_id": "xai__grok-4", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "root:apex_v1_score": 0.635, }, }, { "annotations_by_metric": { "root:apex_v1_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash", "name": "Gemini 2.5 Flash", }, "model_route_id": "google__gemini-2-5-flash", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "root:apex_v1_score": 0.604, }, }, { "annotations_by_metric": { "root:apex_v1_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT 4o", }, "model_route_id": "openai__gpt-4o", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "root:apex_v1_score": 0.359, }, }, { "annotations_by_metric": { "subtask:big_law:apex_v1_big_law_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-1", "name": "GPT 5.1", }, "model_route_id": "openai__gpt-5-1", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "subtask:big_law:apex_v1_big_law_score": 0.77, }, }, { "annotations_by_metric": { "subtask:big_law:apex_v1_big_law_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o3", "name": "o3", }, "model_route_id": "openai__o3", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "subtask:big_law:apex_v1_big_law_score": 0.76, }, }, { "annotations_by_metric": { "subtask:medicine_md:apex_v1_medicine_md_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1773260200", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/opus-4-5", "name": "Opus 4.5", }, "model_route_id": "anthropic__opus-4-5", "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "values": { "subtask:medicine_md:apex_v1_medicine_md_score": 0.65, }, }, ], "metric_config": { "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_names": [ "Score", "Big Law / Score", "Consulting / Score", "Investment Banking / Score", "Medicine (MD) / Score", ], "metrics_count": 5, "missing_generation_config_count": 0, "model_results": [ { "evaluation_timestamp": "1773260200", "model_info": { "developer": "openai", "id": "openai/gpt-5", "name": "GPT 5", }, "model_route_id": "openai__gpt-5", "result": { "canonical_display_name": "APEX v1 / Score", "detailed_evaluation_results_url": undefined, "display_name": "APEX v1 / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1773260200", "metric_config": { "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "apex_v1_score", "score_details": { "score": 0.67, }, }, "score": 0.67, "score_details": { "score": 0.67, }, "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5/apex_v1_openai_gpt_5_1773260200.json", }, { "evaluation_timestamp": "1773260200", "model_info": { "developer": "openai", "id": "openai/gpt-5-2-pro", "name": "GPT 5.2 Pro", }, "model_route_id": "openai__gpt-5-2-pro", "result": { "canonical_display_name": "APEX v1 / Score", "detailed_evaluation_results_url": undefined, "display_name": "APEX v1 / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1773260200", "metric_config": { "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "apex_v1_score", "score_details": { "score": 0.668, }, }, "score": 0.668, "score_details": { "score": 0.668, }, "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2-pro/apex_v1_openai_gpt_5_2_pro_1773260200.json", }, { "evaluation_timestamp": "1773260200", "model_info": { "developer": "google", "id": "google/gemini-3-pro", "name": "Gemini 3 Pro", }, "model_route_id": "google__gemini-3-pro", "result": { "canonical_display_name": "APEX v1 / Score", "detailed_evaluation_results_url": undefined, "display_name": "APEX v1 / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1773260200", "metric_config": { "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "apex_v1_score", "score_details": { "score": 0.643, }, }, "score": 0.643, "score_details": { "score": 0.643, }, "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro/apex_v1_google_gemini_3_pro_1773260200.json", }, { "evaluation_timestamp": "1773260200", "model_info": { "developer": "google", "id": "google/gemini-3-flash", "name": "Gemini 3 Flash", }, "model_route_id": "google__gemini-3-flash", "result": { "canonical_display_name": "APEX v1 / Score", "detailed_evaluation_results_url": undefined, "display_name": "APEX v1 / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1773260200", "metric_config": { "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "apex_v1_score", "score_details": { "score": 0.64, }, }, "score": 0.64, "score_details": { "score": 0.64, }, "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-flash/apex_v1_google_gemini_3_flash_1773260200.json", }, { "evaluation_timestamp": "1773260200", "model_info": { "developer": "xai", "id": "xai/grok-4", "name": "Grok 4", }, "model_route_id": "xai__grok-4", "result": { "canonical_display_name": "APEX v1 / Score", "detailed_evaluation_results_url": undefined, "display_name": "APEX v1 / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1773260200", "metric_config": { "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "apex_v1_score", "score_details": { "score": 0.635, }, }, "score": 0.635, "score_details": { "score": 0.635, }, "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-4/apex_v1_xai_grok_4_1773260200.json", }, { "evaluation_timestamp": "1773260200", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash", "name": "Gemini 2.5 Flash", }, "model_route_id": "google__gemini-2-5-flash", "result": { "canonical_display_name": "APEX v1 / Score", "detailed_evaluation_results_url": undefined, "display_name": "APEX v1 / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1773260200", "metric_config": { "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "apex_v1_score", "score_details": { "score": 0.604, }, }, "score": 0.604, "score_details": { "score": 0.604, }, "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash/apex_v1_google_gemini_2_5_flash_1773260200.json", }, { "evaluation_timestamp": "1773260200", "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT 4o", }, "model_route_id": "openai__gpt-4o", "result": { "canonical_display_name": "APEX v1 / Score", "detailed_evaluation_results_url": undefined, "display_name": "APEX v1 / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": true, "is_multi_source": false, "signal_version": "1.0", "source_type": "first_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1773260200", "metric_config": { "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "apex_v1_score", "score_details": { "score": 0.359, }, }, "score": 0.359, "score_details": { "score": 0.359, }, "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_metadata": { "evaluator_relationship": "first_party", "source_name": "Mercor APEX-v1 Leaderboard", "source_organization_name": "Mercor", "source_organization_url": "https://www.mercor.com", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o/apex_v1_openai_gpt_4o_1773260200.json", }, ], "models_count": 7, "provenance_summary": { "first_party_only_groups": 19, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 19, "third_party": 0, "unspecified": 0, }, "total_groups": 19, "total_results": 19, }, "reproducibility_summary": { "has_reproducibility_gap_count": 19, "populated_ratio_avg": 0, "results_total": 19, }, "root_metrics": [ { "canonical_display_name": "APEX v1 / Score", "display_name": "APEX v1 / Score", "lower_is_better": false, "metric_key": "score", "metric_name": "Score", "metric_summary_id": "apex_v1_score", "models_count": 7, "top_score": 0.67, "unit": "proportion", }, ], "source_data": { "dataset_name": "apex-v1", "hf_repo": "Mercor/APEX-v1", "source_type": "hf_dataset", }, "source_types": [], "subtasks": [ { "canonical_display_name": undefined, "display_name": "Big Law", "metrics": [ { "canonical_display_name": "APEX v1 / Big Law / Score", "display_name": "APEX v1 / Big Law / Score", "lower_is_better": false, "metric_key": "score", "metric_name": "Score", "metric_summary_id": "apex_v1_big_law_score", "models_count": 3, "top_score": 0.78, "unit": "proportion", }, ], "subtask_key": "big_law", "subtask_name": "Big Law", }, { "canonical_display_name": undefined, "display_name": "Consulting", "metrics": [ { "canonical_display_name": "APEX v1 / Consulting / Score", "display_name": "APEX v1 / Consulting / Score", "lower_is_better": false, "metric_key": "score", "metric_name": "Score", "metric_summary_id": "apex_v1_consulting_score", "models_count": 3, "top_score": 0.64, "unit": "proportion", }, ], "subtask_key": "consulting", "subtask_name": "Consulting", }, { "canonical_display_name": undefined, "display_name": "Investment Banking", "metrics": [ { "canonical_display_name": "APEX v1 / Investment Banking / Score", "display_name": "APEX v1 / Investment Banking / Score", "lower_is_better": false, "metric_key": "score", "metric_name": "Score", "metric_summary_id": "apex_v1_investment_banking_score", "models_count": 3, "top_score": 0.64, "unit": "proportion", }, ], "subtask_key": "investment_banking", "subtask_name": "Investment Banking", }, { "canonical_display_name": undefined, "display_name": "Medicine (MD)", "metrics": [ { "canonical_display_name": "APEX v1 / Medicine (MD) / Score", "display_name": "APEX v1 / Medicine (MD) / Score", "lower_is_better": false, "metric_key": "score", "metric_name": "Score", "metric_summary_id": "apex_v1_medicine_md_score", "models_count": 3, "top_score": 0.66, "unit": "proportion", }, ], "subtask_key": "medicine_md", "subtask_name": "Medicine (MD)", }, ], "third_party_ratio": 0, "worst_model": { "name": "GPT 4o", "score": 0.359, }, } `; exports[`hfEvalDetailToSummary > appworld — Pipeline category=coding (one of the 3 keys we added to PIPELINE_CATEGORY_MAP); inferCategoryFromBenchmark returns Agentic 1`] = ` { "avg_score": 0.3805333333333333, "avg_score_norm": 0.3805333333333333, "benchmark_card": { "benchmark_details": { "benchmark_type": "single", "data_type": "text", "domains": [ "autonomous agents", "interactive coding", "tool use", "sequential decision-making", ], "languages": [ "English", ], "name": "AppWorld Benchmark", "overview": "AppWorld is a benchmark designed to evaluate autonomous agents on complex day-to-day digital tasks requiring rich code generation with complex control flow and interaction with multiple applications via APIs. It features 750 natural and diverse tasks evaluated through programmatic state-based unit tests that check for both task completion and absence of collateral damage.", "resources": [ "https://github.com/stonybrooknlp/appworld", "https://arxiv.org/abs/2407.18901", ], "similar_benchmarks": [ "AndroidWorld", "Android in the wild", "WebArena", "Mind2Web", "VisualWebArena", "OSWorld", "API-Bank", "TaskBench", "RestGPT", ], }, "card_info": { "created_at": "2026-04-14T12:15:41.946449", "llm": "deepseek-ai/DeepSeek-V3.1", }, "data": { "annotation": "Tasks were generated by the authors, who undertook meticulous quality control measures including multiple code reviews, documentation, and unit-testing. Some text entries were manually generated or created using ChatGPT for prompts where precise semantics were not critical.", "format": "The data is structured as a suite of tasks with associated database states, API documentation, and programmatic evaluation tests.", "size": "The benchmark consists of 750 tasks, which are split into Train, Dev, Test-N ('normal' set), and Test-C ('challenge' set). Test-C contains tasks that require an API from at least one designated unseen app (Amazon and Gmail).", "source": "The AppWorld benchmark data was entirely developed by the authors over a period of five months, rather than being crowdsourced. It is built upon the AppWorld Engine, which includes 9 day-to-day applications with 457 APIs and is populated with realistic digital activities simulating the lives of approximately 100 fictitious users.", }, "ethical_and_legal_considerations": { "compliance_with_regulations": "No information was provided regarding IRB approval, GDPR compliance, or ethical review. The apps referenced are canonical examples and not intended to imply affiliation or endorsement.", "consent_procedures": "No crowdworkers or annotators were involved as the data was developed by the authors.", "data_licensing": "Not specified", "privacy_and_anonymity": "The benchmark uses simulated fictitious user data for approximately 100 users, ensuring no real personal data is involved. It includes a canary string in the code and data to facilitate filtering from training corpora.", }, "flagged_fields": {}, "methodology": { "baseline_results": "GPT-4O with ReAct: 48.8 TGC on Test-N, 30.2 on Test-C; GPT-4 Trb with ReAct: 32.7 TGC on Test-N, 17.5 on Test-C; LLaMA3 with FullCodeRefl: 24.4 TGC on Test-N, 7.0 on Test-C; CodeAct and ToolLLaMA failed on all tasks", "calculation": "TGC is the percentage of tasks for which the agent passed all evaluation tests. SGC is the percentage of task scenarios for which the agent passed all evaluation tests for all tasks from that scenario. Evaluation uses programmatic assertions to check whether the final database state contains all expected changes and no unexpected changes", "interpretation": "Higher TGC and SGC scores indicate better performance. GPT-4O with ReAct achieved 48.8 TGC on Test-N and 30.2 on Test-C, representing state-of-the-art performance that highlights the benchmark's difficulty", "methods": [ "Models are evaluated using prompting-based agent methods such as ReAct, PlanAndExecute, FullCodeRefl, and IPFunCall", "Evaluation is based on the agent's ability to interact with the environment and generate code to complete tasks", ], "metrics": [ "Task Goal Completion (TGC)", "Scenario Goal Completion (SGC)", ], "validation": "Validation solutions were written by the authors to ensure tasks are solvable, using minimal internal knowledge about data setup. The framework was tested to ensure tasks pass evaluation tests when run with the solution code", }, "missing_fields": [ "ethical_and_legal_considerations.data_licensing", ], "possible_risks": [ { "category": "Over- or under-reliance", "description": [ "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html", }, { "category": "Mitigation and maintenance", "description": [ "The large number of components and dependencies that agent systems have complicates keeping them up to date and correcting problems.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/mitigation-maintenance-agentic.html", }, { "category": "Unrepresentative data", "description": [ "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html", }, { "category": "Reproducibility", "description": [ "Replicating agent behavior or output can be impacted by changes or updates made to external services and tools. This impact is increased if the agent is built with generative AI.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/reproducibility-agentic.html", }, { "category": "Improper usage", "description": [ "Improper usage occurs when a model is used for a purpose that it was not originally designed for.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html", }, ], "purpose_and_intended_users": { "audience": [ "Researchers developing and evaluating autonomous agents", "Developers focused on tool use, interactive coding, and sequential decision-making", ], "goal": "To benchmark interactive coding agents on realistic day-to-day digital tasks that require operating multiple apps via APIs and generating rich code with complex control flow in an iterative manner based on environment interaction.", "limitations": "The benchmark requires substantial engineering effort and was not crowdsourced. Approximately 15% of tasks are question-answering tasks seeking specific answers.", "out_of_scope_uses": [ "Tasks that only require simple sequences of API calls", "Tasks not requiring rich interactive code generation", ], "tasks": [ "Completing natural day-to-day scenarios (e.g., ordering groceries, managing playlists, handling payments)", "Using multiple applications via APIs", "Reasoning over intermediate results", "Handling hurdles during task execution", ], }, }, "best_model": { "name": "claude-opus-4-5", "score": 0.7, }, "canonical_display_name": "Appworld", "category": "Agentic", "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 3, "total_groups": 3, "variant_divergent_count": 3, }, "composite_benchmark_key": "AppWorld Benchmark", "composite_benchmark_name": "AppWorld Benchmark", "evalcards": { "annotations": { "benchmark_comparability": { "cross_party_divergence_groups": [], "variant_divergence_groups": [ { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.08999999999999997, "group_id": "anthropic__claude-opus-4-5__appworld_test_normal_score", "model_route_id": "anthropic__claude-opus-4-5", "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, }, { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, ], }, ], "divergence_magnitude": 0.45199999999999996, "group_id": "google__gemini-3-pro-preview__appworld_test_normal_score", "model_route_id": "google__gemini-3-pro-preview", "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, }, { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.22, "group_id": "openai__gpt-5-2__appworld_test_normal_score", "model_route_id": "openai__gpt-5-2", "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, }, ], }, "reporting_completeness": { "completeness_score": 0.9285714285714286, "field_scores": [ { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.name", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.overview", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.data_type", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.domains", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.languages", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.similar_benchmarks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.resources", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.goal", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.audience", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.tasks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.limitations", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.methods", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.metrics", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.calculation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.interpretation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.baseline_results", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.validation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "score": 1, }, { "coverage_type": "partial", "field_path": "autobenchmarkcard.data", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_type", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_organization_name", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.evaluator_relationship", "score": 1, }, { "coverage_type": "reserved", "field_path": "evalcards.lifecycle_status", "score": 0, }, { "coverage_type": "reserved", "field_path": "evalcards.preregistration_url", "score": 0, }, ], "missing_required_fields": [ "evalcards.lifecycle_status", "evalcards.preregistration_url", ], "partial_fields": [], "signal_version": "1.0", "total_fields_evaluated": 28, }, }, }, "evaluation_id": "appworld", "evaluation_name": "Appworld", "evaluator_names": [], "latest_source_name": "AppWorld Benchmark", "leaderboard_metrics": [ { "canonical_display_name": "Appworld / Test normal / Score", "column_key": "subtask:test_normal:appworld_test_normal_score", "display_name": "Appworld / Test normal / Score", "lower_is_better": false, "metric_name": "Score", "metric_summary_id": "appworld_test_normal_score", "scope": "subtask", "subtask_key": "test_normal", "subtask_name": "Test normal", "unit": "proportion", }, ], "leaderboard_rows": [ { "annotations_by_metric": { "subtask:test_normal:appworld_test_normal_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.08999999999999997, "group_id": "anthropic__claude-opus-4-5__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.7, 0.68, 0.66, 0.64, 0.61, ], "signal_version": "1.0", "this_triple_score": 0.61, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_timestamp": "1774263615.0201504", "metrics_present": 1, "model_info": { "developer": "Anthropic", "id": "anthropic/claude-opus-4-5", "name": "claude-opus-4-5", }, "model_route_id": "anthropic__claude-opus-4-5", "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "values": { "subtask:test_normal:appworld_test_normal_score": 0.61, }, }, { "annotations_by_metric": { "subtask:test_normal:appworld_test_normal_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, ], }, ], "divergence_magnitude": 0.45199999999999996, "group_id": "google__gemini-3-pro-preview__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.582, 0.55, 0.505, 0.36, 0.13, ], "signal_version": "1.0", "this_triple_score": 0.13, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_timestamp": "1774263615.0201504", "metrics_present": 1, "model_info": { "developer": "Google", "id": "google/gemini-3-pro-preview", "name": "gemini-3-pro-preview", }, "model_route_id": "google__gemini-3-pro-preview", "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "values": { "subtask:test_normal:appworld_test_normal_score": 0.13, }, }, { "annotations_by_metric": { "subtask:test_normal:appworld_test_normal_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.22, "group_id": "openai__gpt-5-2__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "2025-12-11", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.22, 0.071, 0, 0, 0, ], "signal_version": "1.0", "this_triple_score": 0, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_timestamp": "1774263615.0201504", "metrics_present": 1, "model_info": { "developer": "OpenAI", "id": "openai/gpt-5-2", "name": "gpt-5.2-2025-12-11", }, "model_route_id": "openai__gpt-5-2", "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "values": { "subtask:test_normal:appworld_test_normal_score": 0, }, }, ], "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_names": [ "Test normal / Score", ], "metrics_count": 1, "missing_generation_config_count": 0, "model_results": [ { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Anthropic", "id": "anthropic/claude-opus-4-5", "name": "claude-opus-4-5", }, "model_route_id": "anthropic__claude-opus-4-5", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.08999999999999997, "group_id": "anthropic__claude-opus-4-5__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.7, 0.68, 0.66, 0.64, 0.61, ], "signal_version": "1.0", "this_triple_score": 0.7, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.7, }, }, "score": 0.7, "score_details": { "score": 0.7, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-opus-4-5/appworld_test_normal_smolagents_code_anthropic_claude_opus_4_5_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Anthropic", "id": "anthropic/claude-opus-4-5", "name": "claude-opus-4-5", }, "model_route_id": "anthropic__claude-opus-4-5", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.08999999999999997, "group_id": "anthropic__claude-opus-4-5__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.7, 0.68, 0.66, 0.64, 0.61, ], "signal_version": "1.0", "this_triple_score": 0.68, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.68, }, }, "score": 0.68, "score_details": { "score": 0.68, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-opus-4-5/appworld_test_normal_openai_solo_anthropic_claude_opus_4_5_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Anthropic", "id": "anthropic/claude-opus-4-5", "name": "claude-opus-4-5", }, "model_route_id": "anthropic__claude-opus-4-5", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.08999999999999997, "group_id": "anthropic__claude-opus-4-5__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.7, 0.68, 0.66, 0.64, 0.61, ], "signal_version": "1.0", "this_triple_score": 0.66, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.66, }, }, "score": 0.66, "score_details": { "score": 0.66, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-opus-4-5/appworld_test_normal_claude_code_cli_anthropic_claude_opus_4_5_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Anthropic", "id": "anthropic/claude-opus-4-5", "name": "claude-opus-4-5", }, "model_route_id": "anthropic__claude-opus-4-5", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.08999999999999997, "group_id": "anthropic__claude-opus-4-5__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.7, 0.68, 0.66, 0.64, 0.61, ], "signal_version": "1.0", "this_triple_score": 0.64, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.64, }, }, "score": 0.64, "score_details": { "score": 0.64, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-opus-4-5/appworld_test_normal_litellm_tool_calling_with_shortlisting_anthropic_claude_opus_4_5_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Anthropic", "id": "anthropic/claude-opus-4-5", "name": "claude-opus-4-5", }, "model_route_id": "anthropic__claude-opus-4-5", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.08999999999999997, "group_id": "anthropic__claude-opus-4-5__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.7, 0.68, 0.66, 0.64, 0.61, ], "signal_version": "1.0", "this_triple_score": 0.61, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.61, }, }, "score": 0.61, "score_details": { "score": 0.61, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-opus-4-5/appworld_test_normal_litellm_tool_calling_anthropic_claude_opus_4_5_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Google", "id": "google/gemini-3-pro-preview", "name": "gemini-3-pro-preview", }, "model_route_id": "google__gemini-3-pro-preview", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, ], }, ], "divergence_magnitude": 0.45199999999999996, "group_id": "google__gemini-3-pro-preview__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.582, 0.55, 0.505, 0.36, 0.13, ], "signal_version": "1.0", "this_triple_score": 0.582, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.582, }, }, "score": 0.582, "score_details": { "score": 0.582, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro-preview/appworld_test_normal_openai_solo_google_gemini_3_pro_preview_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Google", "id": "google/gemini-3-pro-preview", "name": "gemini-3-pro-preview", }, "model_route_id": "google__gemini-3-pro-preview", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, ], }, ], "divergence_magnitude": 0.45199999999999996, "group_id": "google__gemini-3-pro-preview__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.582, 0.55, 0.505, 0.36, 0.13, ], "signal_version": "1.0", "this_triple_score": 0.55, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.55, }, }, "score": 0.55, "score_details": { "score": 0.55, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro-preview/appworld_test_normal_litellm_tool_calling_with_shortlisting_google_gemini_3_pro_preview_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Google", "id": "google/gemini-3-pro-preview", "name": "gemini-3-pro-preview", }, "model_route_id": "google__gemini-3-pro-preview", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, ], }, ], "divergence_magnitude": 0.45199999999999996, "group_id": "google__gemini-3-pro-preview__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.582, 0.55, 0.505, 0.36, 0.13, ], "signal_version": "1.0", "this_triple_score": 0.505, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.505, }, }, "score": 0.505, "score_details": { "score": 0.505, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro-preview/appworld_test_normal_litellm_tool_calling_google_gemini_3_pro_preview_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Google", "id": "google/gemini-3-pro-preview", "name": "gemini-3-pro-preview", }, "model_route_id": "google__gemini-3-pro-preview", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, ], }, ], "divergence_magnitude": 0.45199999999999996, "group_id": "google__gemini-3-pro-preview__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.582, 0.55, 0.505, 0.36, 0.13, ], "signal_version": "1.0", "this_triple_score": 0.36, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.36, }, }, "score": 0.36, "score_details": { "score": 0.36, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro-preview/appworld_test_normal_claude_code_cli_google_gemini_3_pro_preview_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "OpenAI", "id": "openai/gpt-5-2", "name": "gpt-5.2-2025-12-11", }, "model_route_id": "openai__gpt-5-2", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.22, "group_id": "openai__gpt-5-2__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "2025-12-11", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.22, 0.071, 0, 0, 0, ], "signal_version": "1.0", "this_triple_score": 0.22, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.22, }, }, "score": 0.22, "score_details": { "score": 0.22, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2/appworld_test_normal_litellm_tool_calling_with_shortlisting_openai_gpt_5_2_2025_12_11_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "Google", "id": "google/gemini-3-pro-preview", "name": "gemini-3-pro-preview", }, "model_route_id": "google__gemini-3-pro-preview", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, ], }, ], "divergence_magnitude": 0.45199999999999996, "group_id": "google__gemini-3-pro-preview__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "default", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.582, 0.55, 0.505, 0.36, 0.13, ], "signal_version": "1.0", "this_triple_score": 0.13, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.13, }, }, "score": 0.13, "score_details": { "score": 0.13, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro-preview/appworld_test_normal_smolagents_code_google_gemini_3_pro_preview_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "OpenAI", "id": "openai/gpt-5-2", "name": "gpt-5.2-2025-12-11", }, "model_route_id": "openai__gpt-5-2", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.22, "group_id": "openai__gpt-5-2__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "2025-12-11", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.22, 0.071, 0, 0, 0, ], "signal_version": "1.0", "this_triple_score": 0.071, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0.071, }, }, "score": 0.071, "score_details": { "score": 0.071, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2/appworld_test_normal_smolagents_code_openai_gpt_5_2_2025_12_11_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "OpenAI", "id": "openai/gpt-5-2", "name": "gpt-5.2-2025-12-11", }, "model_route_id": "openai__gpt-5-2", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.22, "group_id": "openai__gpt-5-2__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "2025-12-11", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.22, 0.071, 0, 0, 0, ], "signal_version": "1.0", "this_triple_score": 0, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2/appworld_test_normal_openai_solo_openai_gpt_5_2_2025_12_11_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "OpenAI", "id": "openai/gpt-5-2", "name": "gpt-5.2-2025-12-11", }, "model_route_id": "openai__gpt-5-2", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.22, "group_id": "openai__gpt-5-2__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "2025-12-11", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.22, 0.071, 0, 0, 0, ], "signal_version": "1.0", "this_triple_score": 0, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2/appworld_test_normal_claude_code_cli_openai_gpt_5_2_2025_12_11_1774263615_0201504.json", }, { "evaluation_timestamp": "1774263615.0201504", "model_info": { "developer": "OpenAI", "id": "openai/gpt-5-2", "name": "gpt-5.2-2025-12-11", }, "model_route_id": "openai__gpt-5-2", "result": { "canonical_display_name": "Appworld / Test normal / Score", "detailed_evaluation_results_url": undefined, "display_name": "Appworld / Test normal / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", "eval_plan", "eval_limits", ], "populated_field_count": 0, "required_field_count": 4, "signal_version": "1.0", }, "variant_divergence": { "differing_setup_fields": [ { "field": "agentic_eval_config", "values": [ { "additional_details": { "agent_framework": "tool_calling_with_shortlisting", "agent_name": "LiteLLM Tool Calling with Shortlisting", }, }, { "additional_details": { "agent_framework": "smolagents_code", "agent_name": "SmolAgents Code", }, }, { "additional_details": { "agent_framework": "openai_solo", "agent_name": "OpenAI Solo", }, }, { "additional_details": { "agent_framework": "claude_code", "agent_name": "Claude Code CLI", }, }, { "additional_details": { "agent_framework": "tool_calling", "agent_name": "LiteLLM Tool Calling", }, }, ], }, ], "divergence_magnitude": 0.22, "group_id": "openai__gpt-5-2__appworld_test_normal_score", "group_variant_breakdown": [ { "row_count": 5, "variant_key": "2025-12-11", }, ], "has_variant_divergence": true, "score_scale_anomaly": false, "scores_in_group": [ 0.22, 0.071, 0, 0, 0, ], "signal_version": "1.0", "this_triple_score": 0, "threshold_basis": "proportion_or_continuous_normalized", "threshold_used": 0.05, "triple_count_in_group": 5, }, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774263615.0201504", "metric_config": { "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "appworld_test_normal_score", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "Exgentic Open Agent Leaderboard", "source_organization_name": "Exgentic", "source_organization_url": "https://github.com/Exgentic", "source_type": "evaluation_run", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2/appworld_test_normal_litellm_tool_calling_openai_gpt_5_2_2025_12_11_1774263615_0201504.json", }, ], "models_count": 15, "provenance_summary": { "first_party_only_groups": 0, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 0, "third_party": 15, "unspecified": 0, }, "total_groups": 3, "total_results": 15, }, "reproducibility_summary": { "has_reproducibility_gap_count": 15, "populated_ratio_avg": 0, "results_total": 15, }, "root_metrics": [], "source_data": { "dataset_name": "appworld/test_normal", "source_type": "url", "url": [ "https://github.com/Exgentic/exgentic", ], }, "source_types": [], "subtasks": [ { "canonical_display_name": undefined, "display_name": "Test normal", "metrics": [ { "canonical_display_name": "Appworld / Test normal / Score", "display_name": "Appworld / Test normal / Score", "lower_is_better": false, "metric_key": "score", "metric_name": "Score", "metric_summary_id": "appworld_test_normal_score", "models_count": 15, "top_score": 0.7, "unit": "proportion", }, ], "subtask_key": "test_normal", "subtask_name": "Test normal", }, ], "third_party_ratio": 0, "worst_model": { "name": "gpt-5.2-2025-12-11", "score": 0, }, } `; exports[`hfEvalDetailToSummary > artificial_analysis_llms_artificial_analysis_aime — Artificial Analysis third-party badge; source_type=documentation in pipeline 1`] = ` { "avg_score": 0.33557731958762876, "avg_score_norm": 0.33557731958762876, "benchmark_card": undefined, "best_model": { "name": "GPT-5 (high)", "score": 0.957, }, "canonical_display_name": "artificial_analysis.aime", "category": "General", "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 0, "total_groups": 192, "variant_divergent_count": 0, }, "composite_benchmark_key": "Artificial Analysis LLM API", "composite_benchmark_name": "Artificial Analysis LLM API", "evalcards": { "annotations": { "benchmark_comparability": { "cross_party_divergence_groups": [], "variant_divergence_groups": [], }, "reporting_completeness": { "completeness_score": 0.10714285714285714, "field_scores": [ { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.name", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.overview", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.data_type", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.domains", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.languages", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.similar_benchmarks", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.resources", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.goal", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.audience", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.tasks", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.limitations", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.methods", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.metrics", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.calculation", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.interpretation", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.baseline_results", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.validation", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "score": 0, }, { "coverage_type": "partial", "field_path": "autobenchmarkcard.data", "score": 0, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_type", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_organization_name", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.evaluator_relationship", "score": 1, }, { "coverage_type": "reserved", "field_path": "evalcards.lifecycle_status", "score": 0, }, { "coverage_type": "reserved", "field_path": "evalcards.preregistration_url", "score": 0, }, ], "missing_required_fields": [ "autobenchmarkcard.benchmark_details.name", "autobenchmarkcard.benchmark_details.overview", "autobenchmarkcard.benchmark_details.data_type", "autobenchmarkcard.benchmark_details.domains", "autobenchmarkcard.benchmark_details.languages", "autobenchmarkcard.benchmark_details.similar_benchmarks", "autobenchmarkcard.benchmark_details.resources", "autobenchmarkcard.purpose_and_intended_users.goal", "autobenchmarkcard.purpose_and_intended_users.audience", "autobenchmarkcard.purpose_and_intended_users.tasks", "autobenchmarkcard.purpose_and_intended_users.limitations", "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "autobenchmarkcard.methodology.methods", "autobenchmarkcard.methodology.metrics", "autobenchmarkcard.methodology.calculation", "autobenchmarkcard.methodology.interpretation", "autobenchmarkcard.methodology.baseline_results", "autobenchmarkcard.methodology.validation", "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "autobenchmarkcard.data", "evalcards.lifecycle_status", "evalcards.preregistration_url", ], "partial_fields": [], "signal_version": "1.0", "total_fields_evaluated": 28, }, }, }, "evaluation_id": "artificial_analysis_llms_artificial_analysis_aime", "evaluation_name": "artificial_analysis.aime", "evaluator_names": [], "latest_source_name": "Artificial Analysis LLM API", "leaderboard_metrics": [ { "canonical_display_name": "artificial_analysis.aime / AIME", "column_key": "root:artificial_analysis_llms_artificial_analysis_aime_aime", "display_name": "artificial_analysis.aime / AIME", "lower_is_better": false, "metric_name": "AIME", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "scope": "root", "subtask_key": undefined, "subtask_name": undefined, "unit": "proportion", }, ], "leaderboard_rows": [ { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5", "name": "GPT-5 (high)", }, "model_route_id": "openai__gpt-5", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.957, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-4", "name": "Grok 4", }, "model_route_id": "xai__grok-4", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.943, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o4-mini", "name": "o4-mini (high)", }, "model_route_id": "openai__o4-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.94, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-235b-a22b-instruct-2507-reasoning", "name": "Qwen3 235B A22B 2507 (Reasoning)", }, "model_route_id": "alibaba__qwen3-235b-a22b-instruct-2507-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.94, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-3-mini-reasoning", "name": "Grok 3 mini Reasoning (high)", }, "model_route_id": "xai__grok-3-mini-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.933, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-medium", "name": "GPT-5 (medium)", }, "model_route_id": "openai__gpt-5-medium", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.917, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-30b-a3b-2507-reasoning", "name": "Qwen3 30B A3B 2507 (Reasoning)", }, "model_route_id": "alibaba__qwen3-30b-a3b-2507-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.907, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o3", "name": "o3", }, "model_route_id": "openai__o3", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.903, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1", "name": "DeepSeek R1 0528 (May '25)", }, "model_route_id": "deepseek__deepseek-r1", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.893, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-pro", "name": "Gemini 2.5 Pro", }, "model_route_id": "google__gemini-2-5-pro", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.887, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "zai", "id": "zai/glm-4-5", "name": "GLM-4.5 (Reasoning)", }, "model_route_id": "zai__glm-4-5", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.873, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-pro-03-25", "name": "Gemini 2.5 Pro Preview (Mar' 25)", }, "model_route_id": "google__gemini-2-5-pro-03-25", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.87, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o3-mini-high", "name": "o3-mini (high)", }, "model_route_id": "openai__o3-mini-high", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.86, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nvidia", "id": "nvidia/llama-nemotron-super-49b-v1-5-reasoning", "name": "Llama Nemotron Super 49B v1.5 (Reasoning)", }, "model_route_id": "nvidia__llama-nemotron-super-49b-v1-5-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.86, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "minimax", "id": "minimax/minimax-m1-80k", "name": "MiniMax M1 80k", }, "model_route_id": "minimax__minimax-m1-80k", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.847, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "lg", "id": "lg/exaone-4-0-32b-reasoning", "name": "EXAONE 4.0 32B (Reasoning)", }, "model_route_id": "lg__exaone-4-0-32b-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.843, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-pro-05-06", "name": "Gemini 2.5 Pro Preview (May' 25)", }, "model_route_id": "google__gemini-2-5-pro-05-06", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.843, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-reasoning-04-2025", "name": "Gemini 2.5 Flash Preview (Reasoning)", }, "model_route_id": "google__gemini-2-5-flash-reasoning-04-2025", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.843, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-235b-a22b-instruct-reasoning", "name": "Qwen3 235B A22B (Reasoning)", }, "model_route_id": "alibaba__qwen3-235b-a22b-instruct-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.84, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-low", "name": "GPT-5 (low)", }, "model_route_id": "openai__gpt-5-low", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.83, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-reasoning", "name": "Gemini 2.5 Flash (Reasoning)", }, "model_route_id": "google__gemini-2-5-flash-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.823, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "minimax", "id": "minimax/minimax-m1-40k", "name": "MiniMax M1 40k", }, "model_route_id": "minimax__minimax-m1-40k", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.813, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-32b-instruct-reasoning", "name": "Qwen3 32B (Reasoning)", }, "model_route_id": "alibaba__qwen3-32b-instruct-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.807, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "perplexity", "id": "perplexity/sonar-reasoning-pro", "name": "Sonar Reasoning Pro", }, "model_route_id": "perplexity__sonar-reasoning-pro", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.79, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwq-32b", "name": "QwQ 32B", }, "model_route_id": "alibaba__qwq-32b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.78, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-4-sonnet-thinking", "name": "Claude 4 Sonnet (Reasoning)", }, "model_route_id": "anthropic__claude-4-sonnet-thinking", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.773, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "perplexity", "id": "perplexity/sonar-reasoning", "name": "Sonar Reasoning", }, "model_route_id": "perplexity__sonar-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.77, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o3-mini", "name": "o3-mini", }, "model_route_id": "openai__o3-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.77, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-14b-instruct-reasoning", "name": "Qwen3 14B (Reasoning)", }, "model_route_id": "alibaba__qwen3-14b-instruct-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.763, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-4-opus-thinking", "name": "Claude 4 Opus (Reasoning)", }, "model_route_id": "anthropic__claude-4-opus-thinking", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.757, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-30b-a3b-instruct-reasoning", "name": "Qwen3 30B A3B (Reasoning)", }, "model_route_id": "alibaba__qwen3-30b-a3b-instruct-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.753, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-1-nemotron-ultra-253b-v1-reasoning", "name": "Llama 3.1 Nemotron Ultra 253B v1 (Reasoning)", }, "model_route_id": "nvidia__llama-3-1-nemotron-ultra-253b-v1-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.747, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-8b-instruct-reasoning", "name": "Qwen3 8B (Reasoning)", }, "model_route_id": "alibaba__qwen3-8b-instruct-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.747, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-30b-a3b-2507", "name": "Qwen3 30B A3B 2507 Instruct", }, "model_route_id": "alibaba__qwen3-30b-a3b-2507", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.727, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o1", "name": "o1", }, "model_route_id": "openai__o1", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.723, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-235b-a22b-instruct-2507", "name": "Qwen3 235B A22B 2507 Instruct", }, "model_route_id": "alibaba__qwen3-235b-a22b-instruct-2507", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.717, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/magistral-small", "name": "Magistral Small 1", }, "model_route_id": "mistral__magistral-small", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.713, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-1-nemotron-nano-4b-reasoning", "name": "Llama 3.1 Nemotron Nano 4B v1.1 (Reasoning)", }, "model_route_id": "nvidia__llama-3-1-nemotron-nano-4b-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.707, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-lite-reasoning", "name": "Gemini 2.5 Flash-Lite (Reasoning)", }, "model_route_id": "google__gemini-2-5-flash-lite-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.703, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/magistral-medium", "name": "Magistral Medium 1", }, "model_route_id": "mistral__magistral-medium", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.7, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "kimi", "id": "kimi/kimi-k2", "name": "Kimi K2", }, "model_route_id": "kimi__kimi-k2", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.693, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "upstage", "id": "upstage/solar-pro-2-reasoning", "name": "Solar Pro 2 (Reasoning)", }, "model_route_id": "upstage__solar-pro-2-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.69, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-qwen-32b", "name": "DeepSeek R1 Distill Qwen 32B", }, "model_route_id": "deepseek__deepseek-r1-distill-qwen-32b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.687, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-0120", "name": "DeepSeek R1 (Jan '25)", }, "model_route_id": "deepseek__deepseek-r1-0120", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.683, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "zai", "id": "zai/glm-4-5-air", "name": "GLM-4.5-Air", }, "model_route_id": "zai__glm-4-5-air", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.673, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-llama-70b", "name": "DeepSeek R1 Distill Llama 70B", }, "model_route_id": "deepseek__deepseek-r1-distill-llama-70b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.67, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-qwen-14b", "name": "DeepSeek R1 Distill Qwen 14B", }, "model_route_id": "deepseek__deepseek-r1-distill-qwen-14b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.667, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "upstage", "id": "upstage/solar-pro-2-preview-reasoning", "name": "Solar Pro 2 (Preview) (Reasoning)", }, "model_route_id": "upstage__solar-pro-2-preview-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.663, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-4b-instruct-reasoning", "name": "Qwen3 4B (Reasoning)", }, "model_route_id": "alibaba__qwen3-4b-instruct-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.657, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-qwen3-8b", "name": "DeepSeek R1 0528 Qwen3 8B", }, "model_route_id": "deepseek__deepseek-r1-qwen3-8b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.65, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o1-mini", "name": "o1-mini", }, "model_route_id": "openai__o1-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.603, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-3-nemotron-super-49b-reasoning", "name": "Llama 3.3 Nemotron Super 49B v1 (Reasoning)", }, "model_route_id": "nvidia__llama-3-3-nemotron-super-49b-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.583, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-4-opus", "name": "Claude 4 Opus (Non-reasoning)", }, "model_route_id": "anthropic__claude-4-opus", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.563, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-v3-0324", "name": "DeepSeek V3 0324", }, "model_route_id": "deepseek__deepseek-v3-0324", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.52, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "reka-ai", "id": "reka-ai/reka-flash-3", "name": "Reka Flash 3", }, "model_route_id": "reka-ai__reka-flash-3", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.51, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-1-7b-instruct-reasoning", "name": "Qwen3 1.7B (Reasoning)", }, "model_route_id": "alibaba__qwen3-1-7b-instruct-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.51, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-lite", "name": "Gemini 2.5 Flash-Lite (Non-reasoning)", }, "model_route_id": "google__gemini-2-5-flash-lite", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.5, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash", "name": "Gemini 2.5 Flash (Non-reasoning)", }, "model_route_id": "google__gemini-2-5-flash", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.5, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-thinking-exp-0121", "name": "Gemini 2.0 Flash Thinking Experimental (Jan '25)", }, "model_route_id": "google__gemini-2-0-flash-thinking-exp-0121", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.5, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "baidu", "id": "baidu/ernie-4-5-300b-a47b", "name": "ERNIE 4.5 300B A47B", }, "model_route_id": "baidu__ernie-4-5-300b-a47b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.493, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "perplexity", "id": "perplexity/sonar", "name": "Sonar", }, "model_route_id": "perplexity__sonar", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.487, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-7-sonnet-thinking", "name": "Claude 3.7 Sonnet (Reasoning)", }, "model_route_id": "anthropic__claude-3-7-sonnet-thinking", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.487, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-coder-480b-a35b-instruct", "name": "Qwen3 Coder 480B A35B Instruct", }, "model_route_id": "alibaba__qwen3-coder-480b-a35b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.477, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "lg", "id": "lg/exaone-4-0-32b", "name": "EXAONE 4.0 32B (Non-reasoning)", }, "model_route_id": "lg__exaone-4-0-32b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.47, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwq-32b-preview", "name": "QwQ 32B-Preview", }, "model_route_id": "alibaba__qwq-32b-preview", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.453, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-medium-3", "name": "Mistral Medium 3", }, "model_route_id": "mistral__mistral-medium-3", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.44, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1", "name": "GPT-4.1", }, "model_route_id": "openai__gpt-4-1", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.437, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-04-2025", "name": "Gemini 2.5 Flash Preview (Non-reasoning)", }, "model_route_id": "google__gemini-2-5-flash-04-2025", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.433, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1-mini", "name": "GPT-4.1 mini", }, "model_route_id": "openai__gpt-4-1-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.43, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "upstage", "id": "upstage/solar-pro-2", "name": "Solar Pro 2 (Non-reasoning)", }, "model_route_id": "upstage__solar-pro-2", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.407, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-4-sonnet", "name": "Claude 4 Sonnet (Non-reasoning)", }, "model_route_id": "anthropic__claude-4-sonnet", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.407, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-4-maverick", "name": "Llama 4 Maverick", }, "model_route_id": "meta__llama-4-maverick", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.39, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-minimal", "name": "GPT-5 (minimal)", }, "model_route_id": "openai__gpt-5-minimal", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.367, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-pro-experimental-02-05", "name": "Gemini 2.0 Pro Experimental (Feb '25)", }, "model_route_id": "google__gemini-2-0-pro-experimental-02-05", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.36, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-llama-8b", "name": "DeepSeek R1 Distill Llama 8B", }, "model_route_id": "deepseek__deepseek-r1-distill-llama-8b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.333, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-3", "name": "Grok 3", }, "model_route_id": "xai__grok-3", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.33, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash", "name": "Gemini 2.0 Flash (Feb '25)", }, "model_route_id": "google__gemini-2-0-flash", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.33, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o-chatgpt-03-25", "name": "GPT-4o (March 2025, chatgpt-4o-latest)", }, "model_route_id": "openai__gpt-4o-chatgpt-03-25", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.327, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-235b-a22b-instruct", "name": "Qwen3 235B A22B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-235b-a22b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.327, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-small-3-2", "name": "Mistral Small 3.2", }, "model_route_id": "mistral__mistral-small-3-2", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.323, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-lite-preview", "name": "Gemini 2.0 Flash-Lite (Preview)", }, "model_route_id": "google__gemini-2-0-flash-lite-preview", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.303, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-32b-instruct", "name": "Qwen3 32B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-32b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.303, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-3-instruct-70b", "name": "Llama 3.3 Instruct 70B", }, "model_route_id": "meta__llama-3-3-instruct-70b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.3, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-experimental", "name": "Gemini 2.0 Flash (experimental)", }, "model_route_id": "google__gemini-2-0-flash-experimental", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.3, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "upstage", "id": "upstage/solar-pro-2-preview", "name": "Solar Pro 2 (Preview) (Non-reasoning)", }, "model_route_id": "upstage__solar-pro-2-preview", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.297, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-coder-30b-a3b-instruct", "name": "Qwen3 Coder 30B A3B Instruct", }, "model_route_id": "alibaba__qwen3-coder-30b-a3b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.297, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "perplexity", "id": "perplexity/sonar-pro", "name": "Sonar Pro", }, "model_route_id": "perplexity__sonar-pro", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.29, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-4-scout", "name": "Llama 4 Scout", }, "model_route_id": "meta__llama-4-scout", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.283, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-14b-instruct", "name": "Qwen3 14B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-14b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.28, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-lite-001", "name": "Gemini 2.0 Flash-Lite (Feb '25)", }, "model_route_id": "google__gemini-2-0-flash-lite-001", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.277, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-30b-a3b-instruct", "name": "Qwen3 30B A3B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-30b-a3b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.26, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-3-27b", "name": "Gemma 3 27B Instruct", }, "model_route_id": "google__gemma-3-27b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.253, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-v3", "name": "DeepSeek V3 (Dec '24)", }, "model_route_id": "deepseek__deepseek-v3", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.253, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-1-nemotron-instruct-70b", "name": "Llama 3.1 Nemotron Instruct 70B", }, "model_route_id": "nvidia__llama-3-1-nemotron-instruct-70b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.247, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-8b-instruct", "name": "Qwen3 8B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-8b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.243, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1-nano", "name": "GPT-4.1 nano", }, "model_route_id": "openai__gpt-4-1-nano", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.237, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen-2-5-max", "name": "Qwen2.5 Max", }, "model_route_id": "alibaba__qwen-2-5-max", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.233, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-pro", "name": "Gemini 1.5 Pro (Sep '24)", }, "model_route_id": "google__gemini-1-5-pro", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.23, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-7-sonnet", "name": "Claude 3.7 Sonnet (Non-reasoning)", }, "model_route_id": "anthropic__claude-3-7-sonnet", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.223, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-3-12b", "name": "Gemma 3 12B Instruct", }, "model_route_id": "google__gemma-3-12b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.22, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-instruct-405b", "name": "Llama 3.1 Instruct 405B", }, "model_route_id": "meta__llama-3-1-instruct-405b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.213, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-4b-instruct", "name": "Qwen3 4B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-4b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.213, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "sarvam", "id": "sarvam/sarvam-m-reasoning", "name": "Sarvam M (Reasoning)", }, "model_route_id": "sarvam__sarvam-m-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.203, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-3-nemotron-super-49b", "name": "Llama 3.3 Nemotron Super 49B v1 (Non-reasoning)", }, "model_route_id": "nvidia__llama-3-3-nemotron-super-49b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.193, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-flash", "name": "Gemini 1.5 Flash (Sep '24)", }, "model_route_id": "google__gemini-1-5-flash", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.18, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-qwen-1-5b", "name": "DeepSeek R1 Distill Qwen 1.5B", }, "model_route_id": "deepseek__deepseek-r1-distill-qwen-1-5b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.177, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-instruct-70b", "name": "Llama 3.1 Instruct 70B", }, "model_route_id": "meta__llama-3-1-instruct-70b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.173, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "aws", "id": "aws/nova-premier", "name": "Nova Premier", }, "model_route_id": "aws__nova-premier", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.17, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-5-72b-instruct", "name": "Qwen2.5 Instruct 72B", }, "model_route_id": "alibaba__qwen2-5-72b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.16, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-35-sonnet", "name": "Claude 3.5 Sonnet (Oct '24)", }, "model_route_id": "anthropic__claude-35-sonnet", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.157, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o (Nov '24)", }, "model_route_id": "openai__gpt-4o", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.11, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-turbo", "name": "GPT-4 Turbo", }, "model_route_id": "openai__gpt-4-turbo", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.15, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-72b-instruct", "name": "Qwen2 Instruct 72B", }, "model_route_id": "alibaba__qwen2-72b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.147, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "azure", "id": "azure/phi-4", "name": "Phi-4", }, "model_route_id": "azure__phi-4", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.143, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nvidia", "id": "nvidia/llama-nemotron-super-49b-v1-5", "name": "Llama Nemotron Super 49B v1.5 (Non-reasoning)", }, "model_route_id": "nvidia__llama-nemotron-super-49b-v1-5", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.137, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-3n-e4b", "name": "Gemma 3n E4B Instruct", }, "model_route_id": "google__gemma-3n-e4b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.137, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-2-1212", "name": "Grok 2 (Dec '24)", }, "model_route_id": "xai__grok-2-1212", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.133, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "ai2", "id": "ai2/tulu3-405b", "name": "Llama 3.1 Tulu3 405B", }, "model_route_id": "ai2__tulu3-405b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.133, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-saba", "name": "Mistral Saba", }, "model_route_id": "mistral__mistral-saba", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.13, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-5-coder-32b-instruct", "name": "Qwen2.5 Coder Instruct 32B", }, "model_route_id": "alibaba__qwen2-5-coder-32b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.12, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen-turbo", "name": "Qwen2.5 Turbo", }, "model_route_id": "alibaba__qwen-turbo", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.12, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o-mini", "name": "GPT-4o mini", }, "model_route_id": "openai__gpt-4o-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.117, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-large-2", "name": "Mistral Large 2 (Nov '24)", }, "model_route_id": "mistral__mistral-large-2", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.11, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-5-32b-instruct", "name": "Qwen2.5 Instruct 32B", }, "model_route_id": "alibaba__qwen2-5-32b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.11, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-3n-e4b-preview-0520", "name": "Gemma 3n E4B Instruct Preview (May '25)", }, "model_route_id": "google__gemma-3n-e4b-preview-0520", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.107, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "aws", "id": "aws/nova-pro", "name": "Nova Pro", }, "model_route_id": "aws__nova-pro", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.107, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "aws", "id": "aws/nova-lite", "name": "Nova Lite", }, "model_route_id": "aws__nova-lite", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.107, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-beta", "name": "Grok Beta", }, "model_route_id": "xai__grok-beta", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.103, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o-chatgpt", "name": "GPT-4o (ChatGPT)", }, "model_route_id": "openai__gpt-4o-chatgpt", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.103, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-0-6b-instruct-reasoning", "name": "Qwen3 0.6B (Reasoning)", }, "model_route_id": "alibaba__qwen3-0-6b-instruct-reasoning", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.1, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/command-a", "name": "Command A", }, "model_route_id": "cohere__command-a", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.097, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-35-sonnet-june-24", "name": "Claude 3.5 Sonnet (June '24)", }, "model_route_id": "anthropic__claude-35-sonnet-june-24", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.097, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-1-7b-instruct", "name": "Qwen3 1.7B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-1-7b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.097, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-small-3-1", "name": "Mistral Small 3.1", }, "model_route_id": "mistral__mistral-small-3-1", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.093, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-large-2407", "name": "Mistral Large 2 (Jul '24)", }, "model_route_id": "mistral__mistral-large-2407", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.093, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-2-instruct-11b-vision", "name": "Llama 3.2 Instruct 11B (Vision)", }, "model_route_id": "meta__llama-3-2-instruct-11b-vision", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.093, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-may-2024", "name": "Gemini 1.5 Flash (May '24)", }, "model_route_id": "google__gemini-1-5-flash-may-2024", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.093, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "azure", "id": "azure/phi-4-multimodal", "name": "Phi-4 Multimodal Instruct", }, "model_route_id": "azure__phi-4-multimodal", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.093, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-3n-e2b", "name": "Gemma 3n E2B Instruct", }, "model_route_id": "google__gemma-3n-e2b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.09, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-small-3", "name": "Mistral Small 3", }, "model_route_id": "mistral__mistral-small-3", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.08, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-may-2024", "name": "Gemini 1.5 Pro (May '24)", }, "model_route_id": "google__gemini-1-5-pro-may-2024", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.08, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "aws", "id": "aws/nova-micro", "name": "Nova Micro", }, "model_route_id": "aws__nova-micro", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.08, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-instruct-8b", "name": "Llama 3.1 Instruct 8B", }, "model_route_id": "meta__llama-3-1-instruct-8b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.077, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/pixtral-large-2411", "name": "Pixtral Large", }, "model_route_id": "mistral__pixtral-large-2411", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.07, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/devstral-small-2505", "name": "Devstral Small (May '25)", }, "model_route_id": "mistral__devstral-small-2505", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.067, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/devstral-medium", "name": "Devstral Medium", }, "model_route_id": "mistral__devstral-medium", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.067, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-2-instruct-3b", "name": "Llama 3.2 Instruct 3B", }, "model_route_id": "meta__llama-3-2-instruct-3b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.067, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-small", "name": "Mistral Small (Sep '24)", }, "model_route_id": "mistral__mistral-small", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.063, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-3-4b", "name": "Gemma 3 4B Instruct", }, "model_route_id": "google__gemma-3-4b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.063, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-7-large", "name": "Jamba 1.7 Large", }, "model_route_id": "ai21-labs__jamba-1-7-large", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.057, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-5-coder-7b-instruct", "name": "Qwen2.5 Coder Instruct 7B ", }, "model_route_id": "alibaba__qwen2-5-coder-7b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.053, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-2-instruct-90b-vision", "name": "Llama 3.2 Instruct 90B (Vision)", }, "model_route_id": "meta__llama-3-2-instruct-90b-vision", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.05, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nous-research", "id": "nous-research/deephermes-3-mistral-24b-preview", "name": "DeepHermes 3 - Mistral 24B Preview (Non-reasoning)", }, "model_route_id": "nous-research__deephermes-3-mistral-24b-preview", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.047, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "ibm", "id": "ibm/granite-3-3-8b-instruct", "name": "Granite 3.3 8B (Non-reasoning)", }, "model_route_id": "ibm__granite-3-3-8b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.047, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-sonnet", "name": "Claude 3 Sonnet", }, "model_route_id": "anthropic__claude-3-sonnet", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.047, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-6-large", "name": "Jamba 1.6 Large", }, "model_route_id": "ai21-labs__jamba-1-6-large", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.047, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-5-large", "name": "Jamba 1.5 Large", }, "model_route_id": "ai21-labs__jamba-1-5-large", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.047, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "azure", "id": "azure/phi-3-mini", "name": "Phi-3 Mini Instruct 3.8B", }, "model_route_id": "azure__phi-3-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.04, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-medium", "name": "Mistral Medium", }, "model_route_id": "mistral__mistral-medium", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.037, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-8b", "name": "Gemini 1.5 Flash-8B", }, "model_route_id": "google__gemini-1-5-flash-8b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.033, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-opus", "name": "Claude 3 Opus", }, "model_route_id": "anthropic__claude-3-opus", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.033, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-haiku", "name": "Claude 3.5 Haiku", }, "model_route_id": "anthropic__claude-3-5-haiku", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.033, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-21", "name": "Claude 2.1", }, "model_route_id": "anthropic__claude-21", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.033, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-6-mini", "name": "Jamba 1.6 Mini", }, "model_route_id": "ai21-labs__jamba-1-6-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.033, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "databricks", "id": "databricks/dbrx", "name": "DBRX Instruct", }, "model_route_id": "databricks__dbrx", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.03, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "azure", "id": "azure/phi-4-mini", "name": "Phi-4 Mini Instruct", }, "model_route_id": "azure__phi-4-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.03, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nous-research", "id": "nous-research/hermes-3-llama-3-1-70b", "name": "Hermes 3 - Llama-3.1 70B", }, "model_route_id": "nous-research__hermes-3-llama-3-1-70b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.023, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "liquidai", "id": "liquidai/lfm-40b", "name": "LFM 40B", }, "model_route_id": "liquidai__lfm-40b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.023, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-2-chat-13b", "name": "Llama 2 Chat 13B", }, "model_route_id": "meta__llama-2-chat-13b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.017, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-0-6b-instruct", "name": "Qwen3 0.6B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-0-6b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.017, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-7-mini", "name": "Jamba 1.7 Mini", }, "model_route_id": "ai21-labs__jamba-1-7-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.013, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-haiku", "name": "Claude 3 Haiku", }, "model_route_id": "anthropic__claude-3-haiku", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.01, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-5-mini", "name": "Jamba 1.5 Mini", }, "model_route_id": "ai21-labs__jamba-1-5-mini", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.01, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-small-2402", "name": "Mistral Small (Feb '24)", }, "model_route_id": "mistral__mistral-small-2402", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.007, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-0-pro", "name": "Gemini 1.0 Pro", }, "model_route_id": "google__gemini-1-0-pro", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.007, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/command-r-plus-04-2024", "name": "Command-R+ (Apr '24)", }, "model_route_id": "cohere__command-r-plus-04-2024", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.007, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/command-r-03-2024", "name": "Command-R (Mar '24)", }, "model_route_id": "cohere__command-r-03-2024", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.007, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/devstral-small", "name": "Devstral Small (Jul '25)", }, "model_route_id": "mistral__devstral-small", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0.003, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "openchat", "id": "openchat/openchat-35", "name": "OpenChat 3.5 (1210)", }, "model_route_id": "openchat__openchat-35", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "nous-research", "id": "nous-research/deephermes-3-llama-3-1-8b-preview", "name": "DeepHermes 3 - Llama-3.1 8B Preview (Non-reasoning)", }, "model_route_id": "nous-research__deephermes-3-llama-3-1-8b-preview", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mixtral-8x7b-instruct", "name": "Mixtral 8x7B Instruct", }, "model_route_id": "mistral__mixtral-8x7b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-large", "name": "Mistral Large (Feb '24)", }, "model_route_id": "mistral__mistral-large", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-8x22b-instruct", "name": "Mixtral 8x22B Instruct", }, "model_route_id": "mistral__mistral-8x22b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "mistral", "id": "mistral/mistral-7b-instruct", "name": "Mistral 7B Instruct", }, "model_route_id": "mistral__mistral-7b-instruct", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-instruct-8b", "name": "Llama 3 Instruct 8B", }, "model_route_id": "meta__llama-3-instruct-8b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-instruct-70b", "name": "Llama 3 Instruct 70B", }, "model_route_id": "meta__llama-3-instruct-70b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-2-instruct-1b", "name": "Llama 3.2 Instruct 1B", }, "model_route_id": "meta__llama-3-2-instruct-1b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-2-chat-7b", "name": "Llama 2 Chat 7B", }, "model_route_id": "meta__llama-2-chat-7b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-2-chat-70b", "name": "Llama 2 Chat 70B", }, "model_route_id": "meta__llama-2-chat-70b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-3-1b", "name": "Gemma 3 1B Instruct", }, "model_route_id": "google__gemma-3-1b", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-instant", "name": "Claude Instant", }, "model_route_id": "anthropic__claude-instant", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, { "annotations_by_metric": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1775918921.622802", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-2", "name": "Claude 2.0", }, "model_route_id": "anthropic__claude-2", "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "values": { "root:artificial_analysis_llms_artificial_analysis_aime_aime": 0, }, }, ], "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_names": [ "AIME", ], "metrics_count": 1, "missing_generation_config_count": 0, "model_results": [ { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-5", "name": "GPT-5 (high)", }, "model_route_id": "openai__gpt-5", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.957, }, }, "score": 0.957, "score_details": { "score": 0.957, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5/artificial_analysis_llms_openai_gpt_5_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "xai", "id": "xai/grok-4", "name": "Grok 4", }, "model_route_id": "xai__grok-4", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.943, }, }, "score": 0.943, "score_details": { "score": 0.943, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-4/artificial_analysis_llms_xai_grok_4_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/o4-mini", "name": "o4-mini (high)", }, "model_route_id": "openai__o4-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.94, }, }, "score": 0.94, "score_details": { "score": 0.94, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o4-mini/artificial_analysis_llms_openai_o4_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-235b-a22b-instruct-2507-reasoning", "name": "Qwen3 235B A22B 2507 (Reasoning)", }, "model_route_id": "alibaba__qwen3-235b-a22b-instruct-2507-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.94, }, }, "score": 0.94, "score_details": { "score": 0.94, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-235b-a22b-instruct-2507-reasoning/artificial_analysis_llms_alibaba_qwen3_235b_a22b_instruct_2507_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "xai", "id": "xai/grok-3-mini-reasoning", "name": "Grok 3 mini Reasoning (high)", }, "model_route_id": "xai__grok-3-mini-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.933, }, }, "score": 0.933, "score_details": { "score": 0.933, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-3-mini-reasoning/artificial_analysis_llms_xai_grok_3_mini_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-5-medium", "name": "GPT-5 (medium)", }, "model_route_id": "openai__gpt-5-medium", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.917, }, }, "score": 0.917, "score_details": { "score": 0.917, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-medium/artificial_analysis_llms_openai_gpt_5_medium_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-30b-a3b-2507-reasoning", "name": "Qwen3 30B A3B 2507 (Reasoning)", }, "model_route_id": "alibaba__qwen3-30b-a3b-2507-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.907, }, }, "score": 0.907, "score_details": { "score": 0.907, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-30b-a3b-2507-reasoning/artificial_analysis_llms_alibaba_qwen3_30b_a3b_2507_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/o3", "name": "o3", }, "model_route_id": "openai__o3", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.903, }, }, "score": 0.903, "score_details": { "score": 0.903, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o3/artificial_analysis_llms_openai_o3_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1", "name": "DeepSeek R1 0528 (May '25)", }, "model_route_id": "deepseek__deepseek-r1", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.893, }, }, "score": 0.893, "score_details": { "score": 0.893, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-r1/artificial_analysis_llms_deepseek_deepseek_r1_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-5-pro", "name": "Gemini 2.5 Pro", }, "model_route_id": "google__gemini-2-5-pro", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.887, }, }, "score": 0.887, "score_details": { "score": 0.887, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-pro/artificial_analysis_llms_google_gemini_2_5_pro_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "zai", "id": "zai/glm-4-5", "name": "GLM-4.5 (Reasoning)", }, "model_route_id": "zai__glm-4-5", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.873, }, }, "score": 0.873, "score_details": { "score": 0.873, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/zai__glm-4-5/artificial_analysis_llms_zai_glm_4_5_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-5-pro-03-25", "name": "Gemini 2.5 Pro Preview (Mar' 25)", }, "model_route_id": "google__gemini-2-5-pro-03-25", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.87, }, }, "score": 0.87, "score_details": { "score": 0.87, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-pro-03-25/artificial_analysis_llms_google_gemini_2_5_pro_03_25_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/o3-mini-high", "name": "o3-mini (high)", }, "model_route_id": "openai__o3-mini-high", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.86, }, }, "score": 0.86, "score_details": { "score": 0.86, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o3-mini-high/artificial_analysis_llms_openai_o3_mini_high_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nvidia", "id": "nvidia/llama-nemotron-super-49b-v1-5-reasoning", "name": "Llama Nemotron Super 49B v1.5 (Reasoning)", }, "model_route_id": "nvidia__llama-nemotron-super-49b-v1-5-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.86, }, }, "score": 0.86, "score_details": { "score": 0.86, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nvidia__llama-nemotron-super-49b-v1-5-reasoning/artificial_analysis_llms_nvidia_llama_nemotron_super_49b_v1_5_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "minimax", "id": "minimax/minimax-m1-80k", "name": "MiniMax M1 80k", }, "model_route_id": "minimax__minimax-m1-80k", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.847, }, }, "score": 0.847, "score_details": { "score": 0.847, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/minimax__minimax-m1-80k/artificial_analysis_llms_minimax_minimax_m1_80k_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "lg", "id": "lg/exaone-4-0-32b-reasoning", "name": "EXAONE 4.0 32B (Reasoning)", }, "model_route_id": "lg__exaone-4-0-32b-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.843, }, }, "score": 0.843, "score_details": { "score": 0.843, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/lg__exaone-4-0-32b-reasoning/artificial_analysis_llms_lg_exaone_4_0_32b_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-5-pro-05-06", "name": "Gemini 2.5 Pro Preview (May' 25)", }, "model_route_id": "google__gemini-2-5-pro-05-06", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.843, }, }, "score": 0.843, "score_details": { "score": 0.843, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-pro-05-06/artificial_analysis_llms_google_gemini_2_5_pro_05_06_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-reasoning-04-2025", "name": "Gemini 2.5 Flash Preview (Reasoning)", }, "model_route_id": "google__gemini-2-5-flash-reasoning-04-2025", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.843, }, }, "score": 0.843, "score_details": { "score": 0.843, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash-reasoning-04-2025/artificial_analysis_llms_google_gemini_2_5_flash_reasoning_04_2025_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-235b-a22b-instruct-reasoning", "name": "Qwen3 235B A22B (Reasoning)", }, "model_route_id": "alibaba__qwen3-235b-a22b-instruct-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.84, }, }, "score": 0.84, "score_details": { "score": 0.84, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-235b-a22b-instruct-reasoning/artificial_analysis_llms_alibaba_qwen3_235b_a22b_instruct_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-5-low", "name": "GPT-5 (low)", }, "model_route_id": "openai__gpt-5-low", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.83, }, }, "score": 0.83, "score_details": { "score": 0.83, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-low/artificial_analysis_llms_openai_gpt_5_low_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-reasoning", "name": "Gemini 2.5 Flash (Reasoning)", }, "model_route_id": "google__gemini-2-5-flash-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.823, }, }, "score": 0.823, "score_details": { "score": 0.823, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash-reasoning/artificial_analysis_llms_google_gemini_2_5_flash_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "minimax", "id": "minimax/minimax-m1-40k", "name": "MiniMax M1 40k", }, "model_route_id": "minimax__minimax-m1-40k", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.813, }, }, "score": 0.813, "score_details": { "score": 0.813, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/minimax__minimax-m1-40k/artificial_analysis_llms_minimax_minimax_m1_40k_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-32b-instruct-reasoning", "name": "Qwen3 32B (Reasoning)", }, "model_route_id": "alibaba__qwen3-32b-instruct-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.807, }, }, "score": 0.807, "score_details": { "score": 0.807, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-32b-instruct-reasoning/artificial_analysis_llms_alibaba_qwen3_32b_instruct_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "perplexity", "id": "perplexity/sonar-reasoning-pro", "name": "Sonar Reasoning Pro", }, "model_route_id": "perplexity__sonar-reasoning-pro", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.79, }, }, "score": 0.79, "score_details": { "score": 0.79, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/perplexity__sonar-reasoning-pro/artificial_analysis_llms_perplexity_sonar_reasoning_pro_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwq-32b", "name": "QwQ 32B", }, "model_route_id": "alibaba__qwq-32b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.78, }, }, "score": 0.78, "score_details": { "score": 0.78, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwq-32b/artificial_analysis_llms_alibaba_qwq_32b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-4-sonnet-thinking", "name": "Claude 4 Sonnet (Reasoning)", }, "model_route_id": "anthropic__claude-4-sonnet-thinking", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.773, }, }, "score": 0.773, "score_details": { "score": 0.773, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-4-sonnet-thinking/artificial_analysis_llms_anthropic_claude_4_sonnet_thinking_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "perplexity", "id": "perplexity/sonar-reasoning", "name": "Sonar Reasoning", }, "model_route_id": "perplexity__sonar-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.77, }, }, "score": 0.77, "score_details": { "score": 0.77, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/perplexity__sonar-reasoning/artificial_analysis_llms_perplexity_sonar_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/o3-mini", "name": "o3-mini", }, "model_route_id": "openai__o3-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.77, }, }, "score": 0.77, "score_details": { "score": 0.77, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o3-mini/artificial_analysis_llms_openai_o3_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-14b-instruct-reasoning", "name": "Qwen3 14B (Reasoning)", }, "model_route_id": "alibaba__qwen3-14b-instruct-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.763, }, }, "score": 0.763, "score_details": { "score": 0.763, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-14b-instruct-reasoning/artificial_analysis_llms_alibaba_qwen3_14b_instruct_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-4-opus-thinking", "name": "Claude 4 Opus (Reasoning)", }, "model_route_id": "anthropic__claude-4-opus-thinking", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.757, }, }, "score": 0.757, "score_details": { "score": 0.757, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-4-opus-thinking/artificial_analysis_llms_anthropic_claude_4_opus_thinking_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-30b-a3b-instruct-reasoning", "name": "Qwen3 30B A3B (Reasoning)", }, "model_route_id": "alibaba__qwen3-30b-a3b-instruct-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.753, }, }, "score": 0.753, "score_details": { "score": 0.753, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-30b-a3b-instruct-reasoning/artificial_analysis_llms_alibaba_qwen3_30b_a3b_instruct_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-1-nemotron-ultra-253b-v1-reasoning", "name": "Llama 3.1 Nemotron Ultra 253B v1 (Reasoning)", }, "model_route_id": "nvidia__llama-3-1-nemotron-ultra-253b-v1-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.747, }, }, "score": 0.747, "score_details": { "score": 0.747, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nvidia__llama-3-1-nemotron-ultra-253b-v1-reasoning/artificial_analysis_llms_nvidia_llama_3_1_nemotron_ultra_253b_v1_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-8b-instruct-reasoning", "name": "Qwen3 8B (Reasoning)", }, "model_route_id": "alibaba__qwen3-8b-instruct-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.747, }, }, "score": 0.747, "score_details": { "score": 0.747, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-8b-instruct-reasoning/artificial_analysis_llms_alibaba_qwen3_8b_instruct_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-30b-a3b-2507", "name": "Qwen3 30B A3B 2507 Instruct", }, "model_route_id": "alibaba__qwen3-30b-a3b-2507", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.727, }, }, "score": 0.727, "score_details": { "score": 0.727, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-30b-a3b-2507/artificial_analysis_llms_alibaba_qwen3_30b_a3b_2507_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/o1", "name": "o1", }, "model_route_id": "openai__o1", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.723, }, }, "score": 0.723, "score_details": { "score": 0.723, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o1/artificial_analysis_llms_openai_o1_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-235b-a22b-instruct-2507", "name": "Qwen3 235B A22B 2507 Instruct", }, "model_route_id": "alibaba__qwen3-235b-a22b-instruct-2507", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.717, }, }, "score": 0.717, "score_details": { "score": 0.717, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-235b-a22b-instruct-2507/artificial_analysis_llms_alibaba_qwen3_235b_a22b_instruct_2507_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/magistral-small", "name": "Magistral Small 1", }, "model_route_id": "mistral__magistral-small", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.713, }, }, "score": 0.713, "score_details": { "score": 0.713, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__magistral-small/artificial_analysis_llms_mistral_magistral_small_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-1-nemotron-nano-4b-reasoning", "name": "Llama 3.1 Nemotron Nano 4B v1.1 (Reasoning)", }, "model_route_id": "nvidia__llama-3-1-nemotron-nano-4b-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.707, }, }, "score": 0.707, "score_details": { "score": 0.707, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nvidia__llama-3-1-nemotron-nano-4b-reasoning/artificial_analysis_llms_nvidia_llama_3_1_nemotron_nano_4b_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-lite-reasoning", "name": "Gemini 2.5 Flash-Lite (Reasoning)", }, "model_route_id": "google__gemini-2-5-flash-lite-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.703, }, }, "score": 0.703, "score_details": { "score": 0.703, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash-lite-reasoning/artificial_analysis_llms_google_gemini_2_5_flash_lite_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/magistral-medium", "name": "Magistral Medium 1", }, "model_route_id": "mistral__magistral-medium", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.7, }, }, "score": 0.7, "score_details": { "score": 0.7, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__magistral-medium/artificial_analysis_llms_mistral_magistral_medium_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "kimi", "id": "kimi/kimi-k2", "name": "Kimi K2", }, "model_route_id": "kimi__kimi-k2", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.693, }, }, "score": 0.693, "score_details": { "score": 0.693, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/kimi__kimi-k2/artificial_analysis_llms_kimi_kimi_k2_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "upstage", "id": "upstage/solar-pro-2-reasoning", "name": "Solar Pro 2 (Reasoning)", }, "model_route_id": "upstage__solar-pro-2-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.69, }, }, "score": 0.69, "score_details": { "score": 0.69, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/upstage__solar-pro-2-reasoning/artificial_analysis_llms_upstage_solar_pro_2_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-qwen-32b", "name": "DeepSeek R1 Distill Qwen 32B", }, "model_route_id": "deepseek__deepseek-r1-distill-qwen-32b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.687, }, }, "score": 0.687, "score_details": { "score": 0.687, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-r1-distill-qwen-32b/artificial_analysis_llms_deepseek_deepseek_r1_distill_qwen_32b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-0120", "name": "DeepSeek R1 (Jan '25)", }, "model_route_id": "deepseek__deepseek-r1-0120", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.683, }, }, "score": 0.683, "score_details": { "score": 0.683, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-r1-0120/artificial_analysis_llms_deepseek_deepseek_r1_0120_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "zai", "id": "zai/glm-4-5-air", "name": "GLM-4.5-Air", }, "model_route_id": "zai__glm-4-5-air", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.673, }, }, "score": 0.673, "score_details": { "score": 0.673, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/zai__glm-4-5-air/artificial_analysis_llms_zai_glm_4_5_air_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-llama-70b", "name": "DeepSeek R1 Distill Llama 70B", }, "model_route_id": "deepseek__deepseek-r1-distill-llama-70b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.67, }, }, "score": 0.67, "score_details": { "score": 0.67, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-r1-distill-llama-70b/artificial_analysis_llms_deepseek_deepseek_r1_distill_llama_70b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-qwen-14b", "name": "DeepSeek R1 Distill Qwen 14B", }, "model_route_id": "deepseek__deepseek-r1-distill-qwen-14b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.667, }, }, "score": 0.667, "score_details": { "score": 0.667, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-r1-distill-qwen-14b/artificial_analysis_llms_deepseek_deepseek_r1_distill_qwen_14b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "upstage", "id": "upstage/solar-pro-2-preview-reasoning", "name": "Solar Pro 2 (Preview) (Reasoning)", }, "model_route_id": "upstage__solar-pro-2-preview-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.663, }, }, "score": 0.663, "score_details": { "score": 0.663, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/upstage__solar-pro-2-preview-reasoning/artificial_analysis_llms_upstage_solar_pro_2_preview_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-4b-instruct-reasoning", "name": "Qwen3 4B (Reasoning)", }, "model_route_id": "alibaba__qwen3-4b-instruct-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.657, }, }, "score": 0.657, "score_details": { "score": 0.657, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-4b-instruct-reasoning/artificial_analysis_llms_alibaba_qwen3_4b_instruct_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-qwen3-8b", "name": "DeepSeek R1 0528 Qwen3 8B", }, "model_route_id": "deepseek__deepseek-r1-qwen3-8b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.65, }, }, "score": 0.65, "score_details": { "score": 0.65, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-r1-qwen3-8b/artificial_analysis_llms_deepseek_deepseek_r1_qwen3_8b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/o1-mini", "name": "o1-mini", }, "model_route_id": "openai__o1-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.603, }, }, "score": 0.603, "score_details": { "score": 0.603, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o1-mini/artificial_analysis_llms_openai_o1_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-3-nemotron-super-49b-reasoning", "name": "Llama 3.3 Nemotron Super 49B v1 (Reasoning)", }, "model_route_id": "nvidia__llama-3-3-nemotron-super-49b-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.583, }, }, "score": 0.583, "score_details": { "score": 0.583, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nvidia__llama-3-3-nemotron-super-49b-reasoning/artificial_analysis_llms_nvidia_llama_3_3_nemotron_super_49b_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-4-opus", "name": "Claude 4 Opus (Non-reasoning)", }, "model_route_id": "anthropic__claude-4-opus", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.563, }, }, "score": 0.563, "score_details": { "score": 0.563, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-4-opus/artificial_analysis_llms_anthropic_claude_4_opus_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-v3-0324", "name": "DeepSeek V3 0324", }, "model_route_id": "deepseek__deepseek-v3-0324", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.52, }, }, "score": 0.52, "score_details": { "score": 0.52, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-v3-0324/artificial_analysis_llms_deepseek_deepseek_v3_0324_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "reka-ai", "id": "reka-ai/reka-flash-3", "name": "Reka Flash 3", }, "model_route_id": "reka-ai__reka-flash-3", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.51, }, }, "score": 0.51, "score_details": { "score": 0.51, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/reka-ai__reka-flash-3/artificial_analysis_llms_reka_ai_reka_flash_3_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-1-7b-instruct-reasoning", "name": "Qwen3 1.7B (Reasoning)", }, "model_route_id": "alibaba__qwen3-1-7b-instruct-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.51, }, }, "score": 0.51, "score_details": { "score": 0.51, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-1-7b-instruct-reasoning/artificial_analysis_llms_alibaba_qwen3_1_7b_instruct_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-lite", "name": "Gemini 2.5 Flash-Lite (Non-reasoning)", }, "model_route_id": "google__gemini-2-5-flash-lite", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.5, }, }, "score": 0.5, "score_details": { "score": 0.5, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash-lite/artificial_analysis_llms_google_gemini_2_5_flash_lite_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash", "name": "Gemini 2.5 Flash (Non-reasoning)", }, "model_route_id": "google__gemini-2-5-flash", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.5, }, }, "score": 0.5, "score_details": { "score": 0.5, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash/artificial_analysis_llms_google_gemini_2_5_flash_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-thinking-exp-0121", "name": "Gemini 2.0 Flash Thinking Experimental (Jan '25)", }, "model_route_id": "google__gemini-2-0-flash-thinking-exp-0121", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.5, }, }, "score": 0.5, "score_details": { "score": 0.5, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash-thinking-exp-0121/artificial_analysis_llms_google_gemini_2_0_flash_thinking_exp_0121_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "baidu", "id": "baidu/ernie-4-5-300b-a47b", "name": "ERNIE 4.5 300B A47B", }, "model_route_id": "baidu__ernie-4-5-300b-a47b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.493, }, }, "score": 0.493, "score_details": { "score": 0.493, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/baidu__ernie-4-5-300b-a47b/artificial_analysis_llms_baidu_ernie_4_5_300b_a47b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "perplexity", "id": "perplexity/sonar", "name": "Sonar", }, "model_route_id": "perplexity__sonar", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.487, }, }, "score": 0.487, "score_details": { "score": 0.487, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/perplexity__sonar/artificial_analysis_llms_perplexity_sonar_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-7-sonnet-thinking", "name": "Claude 3.7 Sonnet (Reasoning)", }, "model_route_id": "anthropic__claude-3-7-sonnet-thinking", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.487, }, }, "score": 0.487, "score_details": { "score": 0.487, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-7-sonnet-thinking/artificial_analysis_llms_anthropic_claude_3_7_sonnet_thinking_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-coder-480b-a35b-instruct", "name": "Qwen3 Coder 480B A35B Instruct", }, "model_route_id": "alibaba__qwen3-coder-480b-a35b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.477, }, }, "score": 0.477, "score_details": { "score": 0.477, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-coder-480b-a35b-instruct/artificial_analysis_llms_alibaba_qwen3_coder_480b_a35b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "lg", "id": "lg/exaone-4-0-32b", "name": "EXAONE 4.0 32B (Non-reasoning)", }, "model_route_id": "lg__exaone-4-0-32b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.47, }, }, "score": 0.47, "score_details": { "score": 0.47, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/lg__exaone-4-0-32b/artificial_analysis_llms_lg_exaone_4_0_32b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwq-32b-preview", "name": "QwQ 32B-Preview", }, "model_route_id": "alibaba__qwq-32b-preview", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.453, }, }, "score": 0.453, "score_details": { "score": 0.453, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwq-32b-preview/artificial_analysis_llms_alibaba_qwq_32b_preview_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-medium-3", "name": "Mistral Medium 3", }, "model_route_id": "mistral__mistral-medium-3", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.44, }, }, "score": 0.44, "score_details": { "score": 0.44, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-medium-3/artificial_analysis_llms_mistral_mistral_medium_3_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4-1", "name": "GPT-4.1", }, "model_route_id": "openai__gpt-4-1", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.437, }, }, "score": 0.437, "score_details": { "score": 0.437, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1/artificial_analysis_llms_openai_gpt_4_1_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-04-2025", "name": "Gemini 2.5 Flash Preview (Non-reasoning)", }, "model_route_id": "google__gemini-2-5-flash-04-2025", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.433, }, }, "score": 0.433, "score_details": { "score": 0.433, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash-04-2025/artificial_analysis_llms_google_gemini_2_5_flash_04_2025_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4-1-mini", "name": "GPT-4.1 mini", }, "model_route_id": "openai__gpt-4-1-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.43, }, }, "score": 0.43, "score_details": { "score": 0.43, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1-mini/artificial_analysis_llms_openai_gpt_4_1_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "upstage", "id": "upstage/solar-pro-2", "name": "Solar Pro 2 (Non-reasoning)", }, "model_route_id": "upstage__solar-pro-2", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.407, }, }, "score": 0.407, "score_details": { "score": 0.407, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/upstage__solar-pro-2/artificial_analysis_llms_upstage_solar_pro_2_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-4-sonnet", "name": "Claude 4 Sonnet (Non-reasoning)", }, "model_route_id": "anthropic__claude-4-sonnet", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.407, }, }, "score": 0.407, "score_details": { "score": 0.407, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-4-sonnet/artificial_analysis_llms_anthropic_claude_4_sonnet_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-4-maverick", "name": "Llama 4 Maverick", }, "model_route_id": "meta__llama-4-maverick", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.39, }, }, "score": 0.39, "score_details": { "score": 0.39, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-4-maverick/artificial_analysis_llms_meta_llama_4_maverick_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-5-minimal", "name": "GPT-5 (minimal)", }, "model_route_id": "openai__gpt-5-minimal", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.367, }, }, "score": 0.367, "score_details": { "score": 0.367, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-minimal/artificial_analysis_llms_openai_gpt_5_minimal_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-0-pro-experimental-02-05", "name": "Gemini 2.0 Pro Experimental (Feb '25)", }, "model_route_id": "google__gemini-2-0-pro-experimental-02-05", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.36, }, }, "score": 0.36, "score_details": { "score": 0.36, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-pro-experimental-02-05/artificial_analysis_llms_google_gemini_2_0_pro_experimental_02_05_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-llama-8b", "name": "DeepSeek R1 Distill Llama 8B", }, "model_route_id": "deepseek__deepseek-r1-distill-llama-8b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.333, }, }, "score": 0.333, "score_details": { "score": 0.333, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-r1-distill-llama-8b/artificial_analysis_llms_deepseek_deepseek_r1_distill_llama_8b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "xai", "id": "xai/grok-3", "name": "Grok 3", }, "model_route_id": "xai__grok-3", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.33, }, }, "score": 0.33, "score_details": { "score": 0.33, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-3/artificial_analysis_llms_xai_grok_3_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash", "name": "Gemini 2.0 Flash (Feb '25)", }, "model_route_id": "google__gemini-2-0-flash", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.33, }, }, "score": 0.33, "score_details": { "score": 0.33, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash/artificial_analysis_llms_google_gemini_2_0_flash_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4o-chatgpt-03-25", "name": "GPT-4o (March 2025, chatgpt-4o-latest)", }, "model_route_id": "openai__gpt-4o-chatgpt-03-25", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.327, }, }, "score": 0.327, "score_details": { "score": 0.327, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o-chatgpt-03-25/artificial_analysis_llms_openai_gpt_4o_chatgpt_03_25_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-235b-a22b-instruct", "name": "Qwen3 235B A22B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-235b-a22b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.327, }, }, "score": 0.327, "score_details": { "score": 0.327, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-235b-a22b-instruct/artificial_analysis_llms_alibaba_qwen3_235b_a22b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-small-3-2", "name": "Mistral Small 3.2", }, "model_route_id": "mistral__mistral-small-3-2", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.323, }, }, "score": 0.323, "score_details": { "score": 0.323, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-small-3-2/artificial_analysis_llms_mistral_mistral_small_3_2_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-lite-preview", "name": "Gemini 2.0 Flash-Lite (Preview)", }, "model_route_id": "google__gemini-2-0-flash-lite-preview", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.303, }, }, "score": 0.303, "score_details": { "score": 0.303, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash-lite-preview/artificial_analysis_llms_google_gemini_2_0_flash_lite_preview_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-32b-instruct", "name": "Qwen3 32B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-32b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.303, }, }, "score": 0.303, "score_details": { "score": 0.303, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-32b-instruct/artificial_analysis_llms_alibaba_qwen3_32b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-3-instruct-70b", "name": "Llama 3.3 Instruct 70B", }, "model_route_id": "meta__llama-3-3-instruct-70b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.3, }, }, "score": 0.3, "score_details": { "score": 0.3, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-3-instruct-70b/artificial_analysis_llms_meta_llama_3_3_instruct_70b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-experimental", "name": "Gemini 2.0 Flash (experimental)", }, "model_route_id": "google__gemini-2-0-flash-experimental", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.3, }, }, "score": 0.3, "score_details": { "score": 0.3, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash-experimental/artificial_analysis_llms_google_gemini_2_0_flash_experimental_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "upstage", "id": "upstage/solar-pro-2-preview", "name": "Solar Pro 2 (Preview) (Non-reasoning)", }, "model_route_id": "upstage__solar-pro-2-preview", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.297, }, }, "score": 0.297, "score_details": { "score": 0.297, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/upstage__solar-pro-2-preview/artificial_analysis_llms_upstage_solar_pro_2_preview_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-coder-30b-a3b-instruct", "name": "Qwen3 Coder 30B A3B Instruct", }, "model_route_id": "alibaba__qwen3-coder-30b-a3b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.297, }, }, "score": 0.297, "score_details": { "score": 0.297, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-coder-30b-a3b-instruct/artificial_analysis_llms_alibaba_qwen3_coder_30b_a3b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "perplexity", "id": "perplexity/sonar-pro", "name": "Sonar Pro", }, "model_route_id": "perplexity__sonar-pro", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.29, }, }, "score": 0.29, "score_details": { "score": 0.29, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/perplexity__sonar-pro/artificial_analysis_llms_perplexity_sonar_pro_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-4-scout", "name": "Llama 4 Scout", }, "model_route_id": "meta__llama-4-scout", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.283, }, }, "score": 0.283, "score_details": { "score": 0.283, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-4-scout/artificial_analysis_llms_meta_llama_4_scout_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-14b-instruct", "name": "Qwen3 14B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-14b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.28, }, }, "score": 0.28, "score_details": { "score": 0.28, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-14b-instruct/artificial_analysis_llms_alibaba_qwen3_14b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-lite-001", "name": "Gemini 2.0 Flash-Lite (Feb '25)", }, "model_route_id": "google__gemini-2-0-flash-lite-001", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.277, }, }, "score": 0.277, "score_details": { "score": 0.277, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash-lite-001/artificial_analysis_llms_google_gemini_2_0_flash_lite_001_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-30b-a3b-instruct", "name": "Qwen3 30B A3B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-30b-a3b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.26, }, }, "score": 0.26, "score_details": { "score": 0.26, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-30b-a3b-instruct/artificial_analysis_llms_alibaba_qwen3_30b_a3b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemma-3-27b", "name": "Gemma 3 27B Instruct", }, "model_route_id": "google__gemma-3-27b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.253, }, }, "score": 0.253, "score_details": { "score": 0.253, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-3-27b/artificial_analysis_llms_google_gemma_3_27b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-v3", "name": "DeepSeek V3 (Dec '24)", }, "model_route_id": "deepseek__deepseek-v3", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.253, }, }, "score": 0.253, "score_details": { "score": 0.253, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-v3/artificial_analysis_llms_deepseek_deepseek_v3_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-1-nemotron-instruct-70b", "name": "Llama 3.1 Nemotron Instruct 70B", }, "model_route_id": "nvidia__llama-3-1-nemotron-instruct-70b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.247, }, }, "score": 0.247, "score_details": { "score": 0.247, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nvidia__llama-3-1-nemotron-instruct-70b/artificial_analysis_llms_nvidia_llama_3_1_nemotron_instruct_70b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-8b-instruct", "name": "Qwen3 8B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-8b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.243, }, }, "score": 0.243, "score_details": { "score": 0.243, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-8b-instruct/artificial_analysis_llms_alibaba_qwen3_8b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4-1-nano", "name": "GPT-4.1 nano", }, "model_route_id": "openai__gpt-4-1-nano", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.237, }, }, "score": 0.237, "score_details": { "score": 0.237, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1-nano/artificial_analysis_llms_openai_gpt_4_1_nano_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen-2-5-max", "name": "Qwen2.5 Max", }, "model_route_id": "alibaba__qwen-2-5-max", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.233, }, }, "score": 0.233, "score_details": { "score": 0.233, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen-2-5-max/artificial_analysis_llms_alibaba_qwen_2_5_max_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-1-5-pro", "name": "Gemini 1.5 Pro (Sep '24)", }, "model_route_id": "google__gemini-1-5-pro", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.23, }, }, "score": 0.23, "score_details": { "score": 0.23, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-pro/artificial_analysis_llms_google_gemini_1_5_pro_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-7-sonnet", "name": "Claude 3.7 Sonnet (Non-reasoning)", }, "model_route_id": "anthropic__claude-3-7-sonnet", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.223, }, }, "score": 0.223, "score_details": { "score": 0.223, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-7-sonnet/artificial_analysis_llms_anthropic_claude_3_7_sonnet_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemma-3-12b", "name": "Gemma 3 12B Instruct", }, "model_route_id": "google__gemma-3-12b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.22, }, }, "score": 0.22, "score_details": { "score": 0.22, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-3-12b/artificial_analysis_llms_google_gemma_3_12b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-1-instruct-405b", "name": "Llama 3.1 Instruct 405B", }, "model_route_id": "meta__llama-3-1-instruct-405b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.213, }, }, "score": 0.213, "score_details": { "score": 0.213, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-instruct-405b/artificial_analysis_llms_meta_llama_3_1_instruct_405b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-4b-instruct", "name": "Qwen3 4B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-4b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.213, }, }, "score": 0.213, "score_details": { "score": 0.213, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-4b-instruct/artificial_analysis_llms_alibaba_qwen3_4b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "sarvam", "id": "sarvam/sarvam-m-reasoning", "name": "Sarvam M (Reasoning)", }, "model_route_id": "sarvam__sarvam-m-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.203, }, }, "score": 0.203, "score_details": { "score": 0.203, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/sarvam__sarvam-m-reasoning/artificial_analysis_llms_sarvam_sarvam_m_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nvidia", "id": "nvidia/llama-3-3-nemotron-super-49b", "name": "Llama 3.3 Nemotron Super 49B v1 (Non-reasoning)", }, "model_route_id": "nvidia__llama-3-3-nemotron-super-49b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.193, }, }, "score": 0.193, "score_details": { "score": 0.193, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nvidia__llama-3-3-nemotron-super-49b/artificial_analysis_llms_nvidia_llama_3_3_nemotron_super_49b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-1-5-flash", "name": "Gemini 1.5 Flash (Sep '24)", }, "model_route_id": "google__gemini-1-5-flash", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.18, }, }, "score": 0.18, "score_details": { "score": 0.18, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-flash/artificial_analysis_llms_google_gemini_1_5_flash_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "deepseek", "id": "deepseek/deepseek-r1-distill-qwen-1-5b", "name": "DeepSeek R1 Distill Qwen 1.5B", }, "model_route_id": "deepseek__deepseek-r1-distill-qwen-1-5b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.177, }, }, "score": 0.177, "score_details": { "score": 0.177, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek__deepseek-r1-distill-qwen-1-5b/artificial_analysis_llms_deepseek_deepseek_r1_distill_qwen_1_5b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-1-instruct-70b", "name": "Llama 3.1 Instruct 70B", }, "model_route_id": "meta__llama-3-1-instruct-70b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.173, }, }, "score": 0.173, "score_details": { "score": 0.173, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-instruct-70b/artificial_analysis_llms_meta_llama_3_1_instruct_70b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "aws", "id": "aws/nova-premier", "name": "Nova Premier", }, "model_route_id": "aws__nova-premier", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.17, }, }, "score": 0.17, "score_details": { "score": 0.17, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/aws__nova-premier/artificial_analysis_llms_aws_nova_premier_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-5-72b-instruct", "name": "Qwen2.5 Instruct 72B", }, "model_route_id": "alibaba__qwen2-5-72b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.16, }, }, "score": 0.16, "score_details": { "score": 0.16, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen2-5-72b-instruct/artificial_analysis_llms_alibaba_qwen2_5_72b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-35-sonnet", "name": "Claude 3.5 Sonnet (Oct '24)", }, "model_route_id": "anthropic__claude-35-sonnet", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.157, }, }, "score": 0.157, "score_details": { "score": 0.157, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-35-sonnet/artificial_analysis_llms_anthropic_claude_35_sonnet_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o (Nov '24)", }, "model_route_id": "openai__gpt-4o", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.15, }, }, "score": 0.15, "score_details": { "score": 0.15, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o/artificial_analysis_llms_openai_gpt_4o_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4-turbo", "name": "GPT-4 Turbo", }, "model_route_id": "openai__gpt-4-turbo", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.15, }, }, "score": 0.15, "score_details": { "score": 0.15, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-turbo/artificial_analysis_llms_openai_gpt_4_turbo_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-72b-instruct", "name": "Qwen2 Instruct 72B", }, "model_route_id": "alibaba__qwen2-72b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.147, }, }, "score": 0.147, "score_details": { "score": 0.147, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen2-72b-instruct/artificial_analysis_llms_alibaba_qwen2_72b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "azure", "id": "azure/phi-4", "name": "Phi-4", }, "model_route_id": "azure__phi-4", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.143, }, }, "score": 0.143, "score_details": { "score": 0.143, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/azure__phi-4/artificial_analysis_llms_azure_phi_4_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nvidia", "id": "nvidia/llama-nemotron-super-49b-v1-5", "name": "Llama Nemotron Super 49B v1.5 (Non-reasoning)", }, "model_route_id": "nvidia__llama-nemotron-super-49b-v1-5", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.137, }, }, "score": 0.137, "score_details": { "score": 0.137, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nvidia__llama-nemotron-super-49b-v1-5/artificial_analysis_llms_nvidia_llama_nemotron_super_49b_v1_5_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemma-3n-e4b", "name": "Gemma 3n E4B Instruct", }, "model_route_id": "google__gemma-3n-e4b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.137, }, }, "score": 0.137, "score_details": { "score": 0.137, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-3n-e4b/artificial_analysis_llms_google_gemma_3n_e4b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "xai", "id": "xai/grok-2-1212", "name": "Grok 2 (Dec '24)", }, "model_route_id": "xai__grok-2-1212", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.133, }, }, "score": 0.133, "score_details": { "score": 0.133, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-2-1212/artificial_analysis_llms_xai_grok_2_1212_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "ai2", "id": "ai2/tulu3-405b", "name": "Llama 3.1 Tulu3 405B", }, "model_route_id": "ai2__tulu3-405b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.133, }, }, "score": 0.133, "score_details": { "score": 0.133, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai2__tulu3-405b/artificial_analysis_llms_ai2_tulu3_405b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-saba", "name": "Mistral Saba", }, "model_route_id": "mistral__mistral-saba", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.13, }, }, "score": 0.13, "score_details": { "score": 0.13, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-saba/artificial_analysis_llms_mistral_mistral_saba_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-5-coder-32b-instruct", "name": "Qwen2.5 Coder Instruct 32B", }, "model_route_id": "alibaba__qwen2-5-coder-32b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.12, }, }, "score": 0.12, "score_details": { "score": 0.12, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen2-5-coder-32b-instruct/artificial_analysis_llms_alibaba_qwen2_5_coder_32b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen-turbo", "name": "Qwen2.5 Turbo", }, "model_route_id": "alibaba__qwen-turbo", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.12, }, }, "score": 0.12, "score_details": { "score": 0.12, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen-turbo/artificial_analysis_llms_alibaba_qwen_turbo_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4o-mini", "name": "GPT-4o mini", }, "model_route_id": "openai__gpt-4o-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.117, }, }, "score": 0.117, "score_details": { "score": 0.117, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o-mini/artificial_analysis_llms_openai_gpt_4o_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o (Aug '24)", }, "model_route_id": "openai__gpt-4o", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.117, }, }, "score": 0.117, "score_details": { "score": 0.117, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o/artificial_analysis_llms_openai_gpt_4o_2024_08_06_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o (May '24)", }, "model_route_id": "openai__gpt-4o", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.11, }, }, "score": 0.11, "score_details": { "score": 0.11, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o/artificial_analysis_llms_openai_gpt_4o_2024_05_13_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-large-2", "name": "Mistral Large 2 (Nov '24)", }, "model_route_id": "mistral__mistral-large-2", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.11, }, }, "score": 0.11, "score_details": { "score": 0.11, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-large-2/artificial_analysis_llms_mistral_mistral_large_2_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-5-32b-instruct", "name": "Qwen2.5 Instruct 32B", }, "model_route_id": "alibaba__qwen2-5-32b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.11, }, }, "score": 0.11, "score_details": { "score": 0.11, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen2-5-32b-instruct/artificial_analysis_llms_alibaba_qwen2_5_32b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemma-3n-e4b-preview-0520", "name": "Gemma 3n E4B Instruct Preview (May '25)", }, "model_route_id": "google__gemma-3n-e4b-preview-0520", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.107, }, }, "score": 0.107, "score_details": { "score": 0.107, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-3n-e4b-preview-0520/artificial_analysis_llms_google_gemma_3n_e4b_preview_0520_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "aws", "id": "aws/nova-pro", "name": "Nova Pro", }, "model_route_id": "aws__nova-pro", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.107, }, }, "score": 0.107, "score_details": { "score": 0.107, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/aws__nova-pro/artificial_analysis_llms_aws_nova_pro_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "aws", "id": "aws/nova-lite", "name": "Nova Lite", }, "model_route_id": "aws__nova-lite", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.107, }, }, "score": 0.107, "score_details": { "score": 0.107, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/aws__nova-lite/artificial_analysis_llms_aws_nova_lite_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "xai", "id": "xai/grok-beta", "name": "Grok Beta", }, "model_route_id": "xai__grok-beta", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.103, }, }, "score": 0.103, "score_details": { "score": 0.103, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-beta/artificial_analysis_llms_xai_grok_beta_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openai", "id": "openai/gpt-4o-chatgpt", "name": "GPT-4o (ChatGPT)", }, "model_route_id": "openai__gpt-4o-chatgpt", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.103, }, }, "score": 0.103, "score_details": { "score": 0.103, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o-chatgpt/artificial_analysis_llms_openai_gpt_4o_chatgpt_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-0-6b-instruct-reasoning", "name": "Qwen3 0.6B (Reasoning)", }, "model_route_id": "alibaba__qwen3-0-6b-instruct-reasoning", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.1, }, }, "score": 0.1, "score_details": { "score": 0.1, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-0-6b-instruct-reasoning/artificial_analysis_llms_alibaba_qwen3_0_6b_instruct_reasoning_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "cohere", "id": "cohere/command-a", "name": "Command A", }, "model_route_id": "cohere__command-a", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.097, }, }, "score": 0.097, "score_details": { "score": 0.097, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__command-a/artificial_analysis_llms_cohere_command_a_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-35-sonnet-june-24", "name": "Claude 3.5 Sonnet (June '24)", }, "model_route_id": "anthropic__claude-35-sonnet-june-24", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.097, }, }, "score": 0.097, "score_details": { "score": 0.097, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-35-sonnet-june-24/artificial_analysis_llms_anthropic_claude_35_sonnet_june_24_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-1-7b-instruct", "name": "Qwen3 1.7B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-1-7b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.097, }, }, "score": 0.097, "score_details": { "score": 0.097, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-1-7b-instruct/artificial_analysis_llms_alibaba_qwen3_1_7b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-small-3-1", "name": "Mistral Small 3.1", }, "model_route_id": "mistral__mistral-small-3-1", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.093, }, }, "score": 0.093, "score_details": { "score": 0.093, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-small-3-1/artificial_analysis_llms_mistral_mistral_small_3_1_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-large-2407", "name": "Mistral Large 2 (Jul '24)", }, "model_route_id": "mistral__mistral-large-2407", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.093, }, }, "score": 0.093, "score_details": { "score": 0.093, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-large-2407/artificial_analysis_llms_mistral_mistral_large_2407_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-2-instruct-11b-vision", "name": "Llama 3.2 Instruct 11B (Vision)", }, "model_route_id": "meta__llama-3-2-instruct-11b-vision", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.093, }, }, "score": 0.093, "score_details": { "score": 0.093, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-2-instruct-11b-vision/artificial_analysis_llms_meta_llama_3_2_instruct_11b_vision_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-may-2024", "name": "Gemini 1.5 Flash (May '24)", }, "model_route_id": "google__gemini-1-5-flash-may-2024", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.093, }, }, "score": 0.093, "score_details": { "score": 0.093, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-flash-may-2024/artificial_analysis_llms_google_gemini_1_5_flash_may_2024_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "azure", "id": "azure/phi-4-multimodal", "name": "Phi-4 Multimodal Instruct", }, "model_route_id": "azure__phi-4-multimodal", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.093, }, }, "score": 0.093, "score_details": { "score": 0.093, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/azure__phi-4-multimodal/artificial_analysis_llms_azure_phi_4_multimodal_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemma-3n-e2b", "name": "Gemma 3n E2B Instruct", }, "model_route_id": "google__gemma-3n-e2b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.09, }, }, "score": 0.09, "score_details": { "score": 0.09, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-3n-e2b/artificial_analysis_llms_google_gemma_3n_e2b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-small-3", "name": "Mistral Small 3", }, "model_route_id": "mistral__mistral-small-3", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.08, }, }, "score": 0.08, "score_details": { "score": 0.08, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-small-3/artificial_analysis_llms_mistral_mistral_small_3_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-may-2024", "name": "Gemini 1.5 Pro (May '24)", }, "model_route_id": "google__gemini-1-5-pro-may-2024", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.08, }, }, "score": 0.08, "score_details": { "score": 0.08, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-pro-may-2024/artificial_analysis_llms_google_gemini_1_5_pro_may_2024_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "aws", "id": "aws/nova-micro", "name": "Nova Micro", }, "model_route_id": "aws__nova-micro", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.08, }, }, "score": 0.08, "score_details": { "score": 0.08, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/aws__nova-micro/artificial_analysis_llms_aws_nova_micro_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-1-instruct-8b", "name": "Llama 3.1 Instruct 8B", }, "model_route_id": "meta__llama-3-1-instruct-8b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.077, }, }, "score": 0.077, "score_details": { "score": 0.077, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-instruct-8b/artificial_analysis_llms_meta_llama_3_1_instruct_8b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/pixtral-large-2411", "name": "Pixtral Large", }, "model_route_id": "mistral__pixtral-large-2411", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.07, }, }, "score": 0.07, "score_details": { "score": 0.07, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__pixtral-large-2411/artificial_analysis_llms_mistral_pixtral_large_2411_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/devstral-small-2505", "name": "Devstral Small (May '25)", }, "model_route_id": "mistral__devstral-small-2505", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.067, }, }, "score": 0.067, "score_details": { "score": 0.067, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__devstral-small-2505/artificial_analysis_llms_mistral_devstral_small_2505_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/devstral-medium", "name": "Devstral Medium", }, "model_route_id": "mistral__devstral-medium", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.067, }, }, "score": 0.067, "score_details": { "score": 0.067, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__devstral-medium/artificial_analysis_llms_mistral_devstral_medium_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-2-instruct-3b", "name": "Llama 3.2 Instruct 3B", }, "model_route_id": "meta__llama-3-2-instruct-3b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.067, }, }, "score": 0.067, "score_details": { "score": 0.067, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-2-instruct-3b/artificial_analysis_llms_meta_llama_3_2_instruct_3b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-small", "name": "Mistral Small (Sep '24)", }, "model_route_id": "mistral__mistral-small", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.063, }, }, "score": 0.063, "score_details": { "score": 0.063, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-small/artificial_analysis_llms_mistral_mistral_small_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemma-3-4b", "name": "Gemma 3 4B Instruct", }, "model_route_id": "google__gemma-3-4b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.063, }, }, "score": 0.063, "score_details": { "score": 0.063, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-3-4b/artificial_analysis_llms_google_gemma_3_4b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-7-large", "name": "Jamba 1.7 Large", }, "model_route_id": "ai21-labs__jamba-1-7-large", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.057, }, }, "score": 0.057, "score_details": { "score": 0.057, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21-labs__jamba-1-7-large/artificial_analysis_llms_ai21_labs_jamba_1_7_large_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen2-5-coder-7b-instruct", "name": "Qwen2.5 Coder Instruct 7B ", }, "model_route_id": "alibaba__qwen2-5-coder-7b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.053, }, }, "score": 0.053, "score_details": { "score": 0.053, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen2-5-coder-7b-instruct/artificial_analysis_llms_alibaba_qwen2_5_coder_7b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-2-instruct-90b-vision", "name": "Llama 3.2 Instruct 90B (Vision)", }, "model_route_id": "meta__llama-3-2-instruct-90b-vision", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.05, }, }, "score": 0.05, "score_details": { "score": 0.05, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-2-instruct-90b-vision/artificial_analysis_llms_meta_llama_3_2_instruct_90b_vision_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nous-research", "id": "nous-research/deephermes-3-mistral-24b-preview", "name": "DeepHermes 3 - Mistral 24B Preview (Non-reasoning)", }, "model_route_id": "nous-research__deephermes-3-mistral-24b-preview", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.047, }, }, "score": 0.047, "score_details": { "score": 0.047, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nous-research__deephermes-3-mistral-24b-preview/artificial_analysis_llms_nous_research_deephermes_3_mistral_24b_preview_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "ibm", "id": "ibm/granite-3-3-8b-instruct", "name": "Granite 3.3 8B (Non-reasoning)", }, "model_route_id": "ibm__granite-3-3-8b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.047, }, }, "score": 0.047, "score_details": { "score": 0.047, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ibm__granite-3-3-8b-instruct/artificial_analysis_llms_ibm_granite_3_3_8b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-sonnet", "name": "Claude 3 Sonnet", }, "model_route_id": "anthropic__claude-3-sonnet", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.047, }, }, "score": 0.047, "score_details": { "score": 0.047, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-sonnet/artificial_analysis_llms_anthropic_claude_3_sonnet_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-6-large", "name": "Jamba 1.6 Large", }, "model_route_id": "ai21-labs__jamba-1-6-large", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.047, }, }, "score": 0.047, "score_details": { "score": 0.047, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21-labs__jamba-1-6-large/artificial_analysis_llms_ai21_labs_jamba_1_6_large_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-5-large", "name": "Jamba 1.5 Large", }, "model_route_id": "ai21-labs__jamba-1-5-large", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.047, }, }, "score": 0.047, "score_details": { "score": 0.047, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21-labs__jamba-1-5-large/artificial_analysis_llms_ai21_labs_jamba_1_5_large_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "azure", "id": "azure/phi-3-mini", "name": "Phi-3 Mini Instruct 3.8B", }, "model_route_id": "azure__phi-3-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.04, }, }, "score": 0.04, "score_details": { "score": 0.04, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/azure__phi-3-mini/artificial_analysis_llms_azure_phi_3_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-medium", "name": "Mistral Medium", }, "model_route_id": "mistral__mistral-medium", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.037, }, }, "score": 0.037, "score_details": { "score": 0.037, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-medium/artificial_analysis_llms_mistral_mistral_medium_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-8b", "name": "Gemini 1.5 Flash-8B", }, "model_route_id": "google__gemini-1-5-flash-8b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.033, }, }, "score": 0.033, "score_details": { "score": 0.033, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-flash-8b/artificial_analysis_llms_google_gemini_1_5_flash_8b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-opus", "name": "Claude 3 Opus", }, "model_route_id": "anthropic__claude-3-opus", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.033, }, }, "score": 0.033, "score_details": { "score": 0.033, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-opus/artificial_analysis_llms_anthropic_claude_3_opus_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-haiku", "name": "Claude 3.5 Haiku", }, "model_route_id": "anthropic__claude-3-5-haiku", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.033, }, }, "score": 0.033, "score_details": { "score": 0.033, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-5-haiku/artificial_analysis_llms_anthropic_claude_3_5_haiku_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-21", "name": "Claude 2.1", }, "model_route_id": "anthropic__claude-21", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.033, }, }, "score": 0.033, "score_details": { "score": 0.033, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-21/artificial_analysis_llms_anthropic_claude_21_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-6-mini", "name": "Jamba 1.6 Mini", }, "model_route_id": "ai21-labs__jamba-1-6-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.033, }, }, "score": 0.033, "score_details": { "score": 0.033, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21-labs__jamba-1-6-mini/artificial_analysis_llms_ai21_labs_jamba_1_6_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "databricks", "id": "databricks/dbrx", "name": "DBRX Instruct", }, "model_route_id": "databricks__dbrx", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.03, }, }, "score": 0.03, "score_details": { "score": 0.03, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/databricks__dbrx/artificial_analysis_llms_databricks_dbrx_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "azure", "id": "azure/phi-4-mini", "name": "Phi-4 Mini Instruct", }, "model_route_id": "azure__phi-4-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.03, }, }, "score": 0.03, "score_details": { "score": 0.03, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/azure__phi-4-mini/artificial_analysis_llms_azure_phi_4_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nous-research", "id": "nous-research/hermes-3-llama-3-1-70b", "name": "Hermes 3 - Llama-3.1 70B", }, "model_route_id": "nous-research__hermes-3-llama-3-1-70b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.023, }, }, "score": 0.023, "score_details": { "score": 0.023, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nous-research__hermes-3-llama-3-1-70b/artificial_analysis_llms_nous_research_hermes_3_llama_3_1_70b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "liquidai", "id": "liquidai/lfm-40b", "name": "LFM 40B", }, "model_route_id": "liquidai__lfm-40b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.023, }, }, "score": 0.023, "score_details": { "score": 0.023, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/liquidai__lfm-40b/artificial_analysis_llms_liquidai_lfm_40b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-2-chat-13b", "name": "Llama 2 Chat 13B", }, "model_route_id": "meta__llama-2-chat-13b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.017, }, }, "score": 0.017, "score_details": { "score": 0.017, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-2-chat-13b/artificial_analysis_llms_meta_llama_2_chat_13b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "alibaba", "id": "alibaba/qwen3-0-6b-instruct", "name": "Qwen3 0.6B (Non-reasoning)", }, "model_route_id": "alibaba__qwen3-0-6b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.017, }, }, "score": 0.017, "score_details": { "score": 0.017, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alibaba__qwen3-0-6b-instruct/artificial_analysis_llms_alibaba_qwen3_0_6b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-7-mini", "name": "Jamba 1.7 Mini", }, "model_route_id": "ai21-labs__jamba-1-7-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.013, }, }, "score": 0.013, "score_details": { "score": 0.013, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21-labs__jamba-1-7-mini/artificial_analysis_llms_ai21_labs_jamba_1_7_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-haiku", "name": "Claude 3 Haiku", }, "model_route_id": "anthropic__claude-3-haiku", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.01, }, }, "score": 0.01, "score_details": { "score": 0.01, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-haiku/artificial_analysis_llms_anthropic_claude_3_haiku_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "ai21-labs", "id": "ai21-labs/jamba-1-5-mini", "name": "Jamba 1.5 Mini", }, "model_route_id": "ai21-labs__jamba-1-5-mini", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.01, }, }, "score": 0.01, "score_details": { "score": 0.01, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21-labs__jamba-1-5-mini/artificial_analysis_llms_ai21_labs_jamba_1_5_mini_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-small-2402", "name": "Mistral Small (Feb '24)", }, "model_route_id": "mistral__mistral-small-2402", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.007, }, }, "score": 0.007, "score_details": { "score": 0.007, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-small-2402/artificial_analysis_llms_mistral_mistral_small_2402_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemini-1-0-pro", "name": "Gemini 1.0 Pro", }, "model_route_id": "google__gemini-1-0-pro", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.007, }, }, "score": 0.007, "score_details": { "score": 0.007, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-0-pro/artificial_analysis_llms_google_gemini_1_0_pro_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "cohere", "id": "cohere/command-r-plus-04-2024", "name": "Command-R+ (Apr '24)", }, "model_route_id": "cohere__command-r-plus-04-2024", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.007, }, }, "score": 0.007, "score_details": { "score": 0.007, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__command-r-plus-04-2024/artificial_analysis_llms_cohere_command_r_plus_04_2024_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "cohere", "id": "cohere/command-r-03-2024", "name": "Command-R (Mar '24)", }, "model_route_id": "cohere__command-r-03-2024", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.007, }, }, "score": 0.007, "score_details": { "score": 0.007, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__command-r-03-2024/artificial_analysis_llms_cohere_command_r_03_2024_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/devstral-small", "name": "Devstral Small (Jul '25)", }, "model_route_id": "mistral__devstral-small", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0.003, }, }, "score": 0.003, "score_details": { "score": 0.003, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__devstral-small/artificial_analysis_llms_mistral_devstral_small_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "openchat", "id": "openchat/openchat-35", "name": "OpenChat 3.5 (1210)", }, "model_route_id": "openchat__openchat-35", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openchat__openchat-35/artificial_analysis_llms_openchat_openchat_35_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "nous-research", "id": "nous-research/deephermes-3-llama-3-1-8b-preview", "name": "DeepHermes 3 - Llama-3.1 8B Preview (Non-reasoning)", }, "model_route_id": "nous-research__deephermes-3-llama-3-1-8b-preview", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/nous-research__deephermes-3-llama-3-1-8b-preview/artificial_analysis_llms_nous_research_deephermes_3_llama_3_1_8b_preview_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mixtral-8x7b-instruct", "name": "Mixtral 8x7B Instruct", }, "model_route_id": "mistral__mixtral-8x7b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mixtral-8x7b-instruct/artificial_analysis_llms_mistral_mixtral_8x7b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-large", "name": "Mistral Large (Feb '24)", }, "model_route_id": "mistral__mistral-large", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-large/artificial_analysis_llms_mistral_mistral_large_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-8x22b-instruct", "name": "Mixtral 8x22B Instruct", }, "model_route_id": "mistral__mistral-8x22b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-8x22b-instruct/artificial_analysis_llms_mistral_mistral_8x22b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "mistral", "id": "mistral/mistral-7b-instruct", "name": "Mistral 7B Instruct", }, "model_route_id": "mistral__mistral-7b-instruct", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistral__mistral-7b-instruct/artificial_analysis_llms_mistral_mistral_7b_instruct_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-instruct-8b", "name": "Llama 3 Instruct 8B", }, "model_route_id": "meta__llama-3-instruct-8b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-instruct-8b/artificial_analysis_llms_meta_llama_3_instruct_8b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-instruct-70b", "name": "Llama 3 Instruct 70B", }, "model_route_id": "meta__llama-3-instruct-70b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-instruct-70b/artificial_analysis_llms_meta_llama_3_instruct_70b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-3-2-instruct-1b", "name": "Llama 3.2 Instruct 1B", }, "model_route_id": "meta__llama-3-2-instruct-1b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-2-instruct-1b/artificial_analysis_llms_meta_llama_3_2_instruct_1b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-2-chat-7b", "name": "Llama 2 Chat 7B", }, "model_route_id": "meta__llama-2-chat-7b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-2-chat-7b/artificial_analysis_llms_meta_llama_2_chat_7b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "meta", "id": "meta/llama-2-chat-70b", "name": "Llama 2 Chat 70B", }, "model_route_id": "meta__llama-2-chat-70b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-2-chat-70b/artificial_analysis_llms_meta_llama_2_chat_70b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "google", "id": "google/gemma-3-1b", "name": "Gemma 3 1B Instruct", }, "model_route_id": "google__gemma-3-1b", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-3-1b/artificial_analysis_llms_google_gemma_3_1b_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-instant", "name": "Claude Instant", }, "model_route_id": "anthropic__claude-instant", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-instant/artificial_analysis_llms_anthropic_claude_instant_1775918921_622802.json", }, { "evaluation_timestamp": "1775918921.622802", "model_info": { "developer": "anthropic", "id": "anthropic/claude-2", "name": "Claude 2.0", }, "model_route_id": "anthropic__claude-2", "result": { "canonical_display_name": "artificial_analysis.aime / AIME", "detailed_evaluation_results_url": undefined, "display_name": "artificial_analysis.aime / AIME", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "AIME", "evaluation_timestamp": "1775918921.622802", "metric_config": { "evaluation_description": "Benchmark score on AIME.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "aime", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "score_details": { "score": 0, }, }, "score": 0, "score_details": { "score": 0, }, "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_metadata": { "additional_details": { "api_endpoint": "https://artificialanalysis.ai/api/v2/data/llms/models", "api_reference_url": "https://artificialanalysis.ai/api-reference", "attribution_required": "true", "attribution_url": "https://artificialanalysis.ai/", "endpoint_scope": "llms", "methodology_url": "https://artificialanalysis.ai/methodology", "parallel_queries": "1", "prompt_length": "1000", }, "evaluator_relationship": "third_party", "source_name": "Artificial Analysis LLM API", "source_organization_name": "Artificial Analysis", "source_organization_url": "https://artificialanalysis.ai/", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-2/artificial_analysis_llms_anthropic_claude_2_1775918921_622802.json", }, ], "models_count": 194, "provenance_summary": { "first_party_only_groups": 0, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 0, "third_party": 194, "unspecified": 0, }, "total_groups": 192, "total_results": 194, }, "reproducibility_summary": { "has_reproducibility_gap_count": 194, "populated_ratio_avg": 0, "results_total": 194, }, "root_metrics": [ { "canonical_display_name": "artificial_analysis.aime / AIME", "display_name": "artificial_analysis.aime / AIME", "lower_is_better": false, "metric_key": "aime", "metric_name": "AIME", "metric_summary_id": "artificial_analysis_llms_artificial_analysis_aime_aime", "models_count": 194, "top_score": 0.957, "unit": "proportion", }, ], "source_data": { "dataset_name": "Artificial Analysis LLM API", "source_type": "url", "url": [ "https://artificialanalysis.ai/api/v2/data/llms/models", ], }, "source_types": [], "subtasks": [], "third_party_ratio": 0, "worst_model": { "name": "Claude 2.0", "score": 0, }, } `; exports[`hfEvalDetailToSummary > helm_capabilities — Composite parent eval; pipeline category=knowledge 1`] = ` { "avg_score": 0.6325588235294117, "avg_score_norm": 0.6325588235294117, "benchmark_card": { "benchmark_details": { "benchmark_type": "composite", "contains": [ "GPQA", "IFEval", "MMLU-Pro", "Omni-MATH", "WildBench", ], "data_type": "text", "domains": [ "question answering", "information retrieval", "summarization", "sentiment analysis", "toxicity detection", ], "languages": [ "English", ], "name": "Holistic Evaluation of Language Models (HELM)", "overview": "HELM is a comprehensive benchmark suite that measures the capabilities, limitations, and risks of language models across multiple scenarios and metrics. It evaluates models holistically using a multi-metric approach that assesses accuracy, calibration, robustness, fairness, bias, toxicity, and efficiency. The suite combines specialized sub-benchmarks (GPQA, IFEval, MMLU-Pro, Omni-MATH, WildBench) to provide broad coverage across different domains and task types.", "resources": [ "https://crfm.stanford.edu/helm/v0.1.0", "https://github.com/stanford-crfm/helm", "https://arxiv.org/abs/2211.09110", "https://crfm.stanford.edu/helm/capabilities/latest/", "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], "similar_benchmarks": [ "GEM", "XTREME", "GEMv2", ], }, "data": { "annotation": "Annotation processes are specific to each sub-benchmark. Some tasks used crowd-sourced data collection. For human evaluation components, annotators were recruited from Amazon Mechanical Turk, compensated at California minimum wage ($15.00/hr), and each generation was evaluated by three annotators.", "format": "Data is structured according to the specific task of each sub-benchmark. This includes article-summary pairs for summarization and questions paired with multiple-choice answers for question answering.", "size": "The total size varies significantly across the constituent benchmarks. For example, the XSUM dataset contains 204K training, 11K validation, and 11K test examples, while the CNN/DM dataset contains 287K training, 13K validation, and 11K test examples.", "source": "The suite aggregates data from multiple specialized benchmarks, each with its own distinct source. These include datasets created from BBC articles (XSUM), CNN and Daily Mail articles (CNN/DM), and other platforms like Hugging Face and AI2.", }, "ethical_and_legal_considerations": { "compliance_with_regulations": "Not specified", "consent_procedures": "Annotators were recruited from Amazon Mechanical Turk and compensated at a rate based on California minimum wage ($15.00/hr). They were paid $5.50 per task, which was estimated based on the time it took researchers to perform the task themselves.", "data_licensing": "Not specified", "privacy_and_anonymity": "Not specified", }, "flagged_fields": {}, "methodology": { "baseline_results": "The paper reports results for 30 prominent language models, but specific model names and scores for this benchmark suite are not provided", "calculation": "The overall score is presented as a matrix for each model, with scores reported for each scenario and metric combination", "interpretation": "Not specified", "methods": [ "The benchmark evaluates models primarily in a few-shot setting using a fixed set of training examples for all test instances to reflect true few-shot conditions", "The evaluation process is repeated three times to measure variance", ], "metrics": [ "Accuracy", "Calibration", "Robustness", "Fairness", "Bias", "Toxicity", "Efficiency", ], "validation": "For human evaluation, quality assurance was performed using three annotators per generation and inserting 'secret words' in instructions to confirm annotators read them. Standard dataset splits (train/dev/test) and pre-processing practices from prior work are used", }, "missing_fields": [], "possible_risks": [ { "category": "Over- or under-reliance", "description": [ "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html", }, { "category": "Unrepresentative data", "description": [ "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html", }, { "category": "Uncertain data provenance", "description": [ "Data provenance refers to the traceability of data (including synthetic data), which includes its ownership, origin, transformations, and generation. Proving that the data is the same as the original source with correct usage terms is difficult without standardized methods for verifying data sources or generation.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-provenance.html", }, { "category": "Data contamination", "description": [ "Data contamination occurs when incorrect data is used for training. For example, data that is not aligned with model's purpose or data that is already set aside for other development tasks such as testing and evaluation.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-contamination.html", }, { "category": "Reproducibility", "description": [ "Replicating agent behavior or output can be impacted by changes or updates made to external services and tools. This impact is increased if the agent is built with generative AI.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/reproducibility-agentic.html", }, ], "purpose_and_intended_users": { "audience": [ "The AI research community", "Researchers and developers interested in understanding and evaluating language models", ], "goal": "To holistically evaluate the capabilities, limitations, and risks of language models across a wide range of scenarios and metrics, with the primary research objective of improving model transparency.", "limitations": "The benchmark lacks coverage for certain scenarios, such as question answering for neglected English dialects and metrics for trustworthiness. Its selection of 'canonical' tasks involves subjectivity, and it does not cover the full long tail of potential use cases.", "out_of_scope_uses": [ "Multimodal tasks", "Machine translation", "Interactive tasks like dialogue", ], "tasks": [ "Question answering", "Information retrieval", "Summarization", "Sentiment analysis", "Toxicity detection", ], }, }, "best_model": { "name": "GPT-5 mini 2025-08-07", "score": 0.819, }, "canonical_display_name": "Holistic Evaluation of Language Models (HELM)", "category": "General", "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 0, "total_groups": 66, "variant_divergent_count": 0, }, "composite_benchmark_key": "Holistic Evaluation of Language Models (HELM)", "composite_benchmark_name": "Holistic Evaluation of Language Models (HELM)", "evalcards": { "annotations": { "benchmark_comparability": { "cross_party_divergence_groups": [], "variant_divergence_groups": [], }, "reporting_completeness": { "completeness_score": 0.9285714285714286, "field_scores": [ { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.name", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.overview", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.data_type", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.domains", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.languages", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.similar_benchmarks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.resources", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.goal", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.audience", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.tasks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.limitations", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.methods", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.metrics", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.calculation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.interpretation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.baseline_results", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.validation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "score": 1, }, { "coverage_type": "partial", "field_path": "autobenchmarkcard.data", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_type", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_organization_name", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.evaluator_relationship", "score": 1, }, { "coverage_type": "reserved", "field_path": "evalcards.lifecycle_status", "score": 0, }, { "coverage_type": "reserved", "field_path": "evalcards.preregistration_url", "score": 0, }, ], "missing_required_fields": [ "evalcards.lifecycle_status", "evalcards.preregistration_url", ], "partial_fields": [], "signal_version": "1.0", "total_fields_evaluated": 28, }, }, }, "evaluation_id": "helm_capabilities", "evaluation_name": "Holistic Evaluation of Language Models (HELM)", "evaluator_names": [], "latest_source_name": "Holistic Evaluation of Language Models (HELM)", "leaderboard_metrics": [ { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "column_key": "root:helm_capabilities_score", "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "lower_is_better": false, "metric_name": "Score", "metric_summary_id": "helm_capabilities_score", "scope": "root", "subtask_key": undefined, "subtask_name": undefined, "unit": "proportion", }, ], "leaderboard_rows": [ { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-mini", "name": "GPT-5 mini 2025-08-07", }, "model_route_id": "openai__gpt-5-mini", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.819, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o4-mini", "name": "o4-mini 2025-04-16", }, "model_route_id": "openai__o4-mini", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.812, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o3", "name": "o3 2025-04-16", }, "model_route_id": "openai__o3", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.811, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5", "name": "GPT-5 2025-08-07", }, "model_route_id": "openai__gpt-5", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.807, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1776204744.2690232", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-3-pro-preview", "name": "Gemini 3 Pro Preview", }, "model_route_id": "google__gemini-3-pro-preview", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.799, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen3-235b-a22b-instruct-2507-fp8", "name": "Qwen3 235B A22B Instruct 2507 FP8", }, "model_route_id": "qwen__qwen3-235b-a22b-instruct-2507-fp8", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.798, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-4-0709", "name": "Grok 4 0709", }, "model_route_id": "xai__grok-4-0709", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.785, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-opus-4", "name": "Claude 4 Opus 20250514, extended thinking", }, "model_route_id": "anthropic__claude-opus-4", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.757, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-oss-120b", "name": "gpt-oss-120b", }, "model_route_id": "openai__gpt-oss-120b", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.77, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "moonshotai", "id": "moonshotai/kimi-k2-instruct", "name": "Kimi K2 Instruct", }, "model_route_id": "moonshotai__kimi-k2-instruct", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.768, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4", "name": "Claude 4 Sonnet 20250514, extended thinking", }, "model_route_id": "anthropic__claude-sonnet-4", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.733, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1776204744.2690232", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4-5", "name": "Claude 4.5 Sonnet 20250929", }, "model_route_id": "anthropic__claude-sonnet-4-5", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.762, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-nano", "name": "GPT-5 nano 2025-08-07", }, "model_route_id": "openai__gpt-5-nano", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.748, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-pro-preview-03-25", "name": "Gemini 2.5 Pro 03-25 preview", }, "model_route_id": "google__gemini-2-5-pro-preview-03-25", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.745, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-3-beta", "name": "Grok 3 Beta", }, "model_route_id": "xai__grok-3-beta", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.727, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1", "name": "GPT-4.1 2025-04-14", }, "model_route_id": "openai__gpt-4-1", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.727, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen3-235b-a22b-fp8-tput", "name": "Qwen3 235B A22B FP8 Throughput", }, "model_route_id": "qwen__qwen3-235b-a22b-fp8-tput", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.726, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1-mini", "name": "GPT-4.1 mini 2025-04-14", }, "model_route_id": "openai__gpt-4-1-mini", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.726, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-4-maverick-17b-128e-instruct-fp8", "name": "Llama 4 Maverick 17Bx128E Instruct FP8", }, "model_route_id": "meta__llama-4-maverick-17b-128e-instruct-fp8", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.718, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1776204744.2690232", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-haiku-4-5", "name": "Claude 4.5 Haiku 20251001", }, "model_route_id": "anthropic__claude-haiku-4-5", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.717, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1776204744.2690232", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen3-next-80b-a3b-thinking", "name": "Qwen3-Next 80B A3B Thinking", }, "model_route_id": "qwen__qwen3-next-80b-a3b-thinking", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.7, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-r1-0528", "name": "DeepSeek-R1-0528", }, "model_route_id": "deepseek-ai__deepseek-r1-0528", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.699, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-x5", "name": "Palmyra X5", }, "model_route_id": "writer__palmyra-x5", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.696, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-3-mini-beta", "name": "Grok 3 mini Beta", }, "model_route_id": "xai__grok-3-mini-beta", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.679, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-001", "name": "Gemini 2.0 Flash", }, "model_route_id": "google__gemini-2-0-flash-001", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.679, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-oss-20b", "name": "gpt-oss-20b", }, "model_route_id": "openai__gpt-oss-20b", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.674, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-7-sonnet", "name": "Claude 3.7 Sonnet 20250219", }, "model_route_id": "anthropic__claude-3-7-sonnet", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.674, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "zai-org", "id": "zai-org/glm-4-5-air-fp8", "name": "GLM-4.5-Air-FP8", }, "model_route_id": "zai-org__glm-4-5-air-fp8", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.67, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-v3", "name": "DeepSeek v3", }, "model_route_id": "deepseek-ai__deepseek-v3", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.665, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-002", "name": "Gemini 1.5 Pro 002", }, "model_route_id": "google__gemini-1-5-pro-002", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.657, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1776204744.2690232", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-1", "name": "GPT-5.1 2025-11-13", }, "model_route_id": "openai__gpt-5-1", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.656, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-sonnet", "name": "Claude 3.5 Sonnet 20241022", }, "model_route_id": "anthropic__claude-3-5-sonnet", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.653, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-4-scout-17b-16e-instruct", "name": "Llama 4 Scout 17Bx16E Instruct", }, "model_route_id": "meta__llama-4-scout-17b-16e-instruct", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.644, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-lite-preview-02-05", "name": "Gemini 2.0 Flash Lite 02-05 preview", }, "model_route_id": "google__gemini-2-0-flash-lite-preview-02-05", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.642, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "amazon", "id": "amazon/nova-premier-v1-0", "name": "Amazon Nova Premier", }, "model_route_id": "amazon__nova-premier-v1-0", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.637, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o 2024-11-20", }, "model_route_id": "openai__gpt-4o", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.634, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-preview-04-17", "name": "Gemini 2.5 Flash 04-17 preview", }, "model_route_id": "google__gemini-2-5-flash-preview-04-17", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.626, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-405b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 405B", }, "model_route_id": "meta__llama-3-1-405b-instruct-turbo", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.618, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1-nano", "name": "GPT-4.1 nano 2025-04-14", }, "model_route_id": "openai__gpt-4-1-nano", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.616, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-x-004", "name": "Palmyra-X-004", }, "model_route_id": "writer__palmyra-x-004", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.609, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-002", "name": "Gemini 1.5 Flash 002", }, "model_route_id": "google__gemini-1-5-flash-002", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.609, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-72b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 72B", }, "model_route_id": "qwen__qwen2-5-72b-instruct-turbo", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.599, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-large-2411", "name": "Mistral Large 2411", }, "model_route_id": "mistralai__mistral-large-2411", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.598, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-lite", "name": "Gemini 2.5 Flash-Lite", }, "model_route_id": "google__gemini-2-5-flash-lite", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.591, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "amazon", "id": "amazon/nova-pro-v1-0", "name": "Amazon Nova Pro", }, "model_route_id": "amazon__nova-pro-v1-0", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.591, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-fin", "name": "Palmyra Fin", }, "model_route_id": "writer__palmyra-fin", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.577, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1776204744.2690232", "metrics_present": 1, "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-h-small", "name": "IBM Granite 4.0 Small", }, "model_route_id": "ibm__granite-4-0-h-small", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.575, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-70b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 70B", }, "model_route_id": "meta__llama-3-1-70b-instruct-turbo", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.574, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o-mini", "name": "GPT-4o mini 2024-07-18", }, "model_route_id": "openai__gpt-4o-mini", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.565, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-small-2503", "name": "Mistral Small 3.1 2503", }, "model_route_id": "mistralai__mistral-small-2503", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.558, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "amazon", "id": "amazon/nova-lite-v1-0", "name": "Amazon Nova Lite", }, "model_route_id": "amazon__nova-lite-v1-0", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.551, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-haiku", "name": "Claude 3.5 Haiku 20241022", }, "model_route_id": "anthropic__claude-3-5-haiku", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.549, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-7b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 7B", }, "model_route_id": "qwen__qwen2-5-7b-instruct-turbo", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.529, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "amazon", "id": "amazon/nova-micro-v1-0", "name": "Amazon Nova Micro", }, "model_route_id": "amazon__nova-micro-v1-0", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.522, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1776204744.2690232", "metrics_present": 1, "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-micro", "name": "IBM Granite 4.0 Micro", }, "model_route_id": "ibm__granite-4-0-micro", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.486, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x22b-instruct-v0-1", "name": "Mixtral Instruct 8x22B", }, "model_route_id": "mistralai__mixtral-8x22b-instruct-v0-1", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.478, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-med", "name": "Palmyra Med", }, "model_route_id": "writer__palmyra-med", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.476, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "allenai", "id": "allenai/olmo-2-0325-32b-instruct", "name": "OLMo 2 32B Instruct March 2025", }, "model_route_id": "allenai__olmo-2-0325-32b-instruct", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.475, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "ibm", "id": "ibm/granite-3-3-8b-instruct", "name": "IBM Granite 3.3 8B Instruct", }, "model_route_id": "ibm__granite-3-3-8b-instruct", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.463, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-8b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 8B", }, "model_route_id": "meta__llama-3-1-8b-instruct-turbo", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.444, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "allenai", "id": "allenai/olmo-2-1124-13b-instruct", "name": "OLMo 2 13B Instruct November 2024", }, "model_route_id": "allenai__olmo-2-1124-13b-instruct", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.44, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "allenai", "id": "allenai/olmo-2-1124-7b-instruct", "name": "OLMo 2 7B Instruct November 2024", }, "model_route_id": "allenai__olmo-2-1124-7b-instruct", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.405, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x7b-instruct-v0-1", "name": "Mixtral Instruct 8x7B", }, "model_route_id": "mistralai__mixtral-8x7b-instruct-v0-1", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.397, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-instruct-v0-3", "name": "Mistral Instruct v0.3 7B", }, "model_route_id": "mistralai__mistral-7b-instruct-v0-3", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.376, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "allenai", "id": "allenai/olmoe-1b-7b-0125-instruct", "name": "OLMoE 1B-7B Instruct January 2025", }, "model_route_id": "allenai__olmoe-1b-7b-0125-instruct", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.332, }, }, { "annotations_by_metric": { "root:helm_capabilities_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096304.5056698", "metrics_present": 1, "model_info": { "developer": "marin-community", "id": "marin-community/marin-8b-instruct", "name": "Marin 8B Instruct", }, "model_route_id": "marin-community__marin-8b-instruct", "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_capabilities_score": 0.325, }, }, ], "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_names": [ "Score", ], "metrics_count": 1, "missing_generation_config_count": 0, "model_results": [ { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-5-mini", "name": "GPT-5 mini 2025-08-07", }, "model_route_id": "openai__gpt-5-mini", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.819, }, }, "score": 0.819, "score_details": { "score": 0.819, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-mini/helm_capabilities_openai_gpt_5_mini_2025_08_07_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/o4-mini", "name": "o4-mini 2025-04-16", }, "model_route_id": "openai__o4-mini", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.812, }, }, "score": 0.812, "score_details": { "score": 0.812, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o4-mini/helm_capabilities_openai_o4_mini_2025_04_16_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/o3", "name": "o3 2025-04-16", }, "model_route_id": "openai__o3", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.811, }, }, "score": 0.811, "score_details": { "score": 0.811, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o3/helm_capabilities_openai_o3_2025_04_16_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-5", "name": "GPT-5 2025-08-07", }, "model_route_id": "openai__gpt-5", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.807, }, }, "score": 0.807, "score_details": { "score": 0.807, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5/helm_capabilities_openai_gpt_5_2025_08_07_1774096304_5056698.json", }, { "evaluation_timestamp": "1776204744.2690232", "model_info": { "developer": "google", "id": "google/gemini-3-pro-preview", "name": "Gemini 3 Pro Preview", }, "model_route_id": "google__gemini-3-pro-preview", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1776204744.2690232", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.799, }, }, "score": 0.799, "score_details": { "score": 0.799, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro-preview/helm_capabilities_google_gemini_3_pro_preview_1776204744_2690232.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "qwen", "id": "qwen/qwen3-235b-a22b-instruct-2507-fp8", "name": "Qwen3 235B A22B Instruct 2507 FP8", }, "model_route_id": "qwen__qwen3-235b-a22b-instruct-2507-fp8", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.798, }, }, "score": 0.798, "score_details": { "score": 0.798, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen3-235b-a22b-instruct-2507-fp8/helm_capabilities_qwen_qwen3_235b_a22b_instruct_2507_fp8_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "xai", "id": "xai/grok-4-0709", "name": "Grok 4 0709", }, "model_route_id": "xai__grok-4-0709", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.785, }, }, "score": 0.785, "score_details": { "score": 0.785, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-4-0709/helm_capabilities_xai_grok_4_0709_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "anthropic", "id": "anthropic/claude-opus-4", "name": "Claude 4 Opus 20250514, extended thinking", }, "model_route_id": "anthropic__claude-opus-4", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.78, }, }, "score": 0.78, "score_details": { "score": 0.78, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-opus-4/helm_capabilities_anthropic_claude_opus_4_20250514_thinking_10k_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-oss-120b", "name": "gpt-oss-120b", }, "model_route_id": "openai__gpt-oss-120b", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.77, }, }, "score": 0.77, "score_details": { "score": 0.77, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-oss-120b/helm_capabilities_openai_gpt_oss_120b_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "moonshotai", "id": "moonshotai/kimi-k2-instruct", "name": "Kimi K2 Instruct", }, "model_route_id": "moonshotai__kimi-k2-instruct", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.768, }, }, "score": 0.768, "score_details": { "score": 0.768, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/moonshotai__kimi-k2-instruct/helm_capabilities_moonshotai_kimi_k2_instruct_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4", "name": "Claude 4 Sonnet 20250514, extended thinking", }, "model_route_id": "anthropic__claude-sonnet-4", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.766, }, }, "score": 0.766, "score_details": { "score": 0.766, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-sonnet-4/helm_capabilities_anthropic_claude_sonnet_4_20250514_thinking_10k_1774096304_5056698.json", }, { "evaluation_timestamp": "1776204744.2690232", "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4-5", "name": "Claude 4.5 Sonnet 20250929", }, "model_route_id": "anthropic__claude-sonnet-4-5", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1776204744.2690232", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.762, }, }, "score": 0.762, "score_details": { "score": 0.762, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-sonnet-4-5/helm_capabilities_anthropic_claude_sonnet_4_5_20250929_1776204744_2690232.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "anthropic", "id": "anthropic/claude-opus-4", "name": "Claude 4 Opus 20250514", }, "model_route_id": "anthropic__claude-opus-4", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.757, }, }, "score": 0.757, "score_details": { "score": 0.757, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-opus-4/helm_capabilities_anthropic_claude_opus_4_20250514_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-5-nano", "name": "GPT-5 nano 2025-08-07", }, "model_route_id": "openai__gpt-5-nano", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.748, }, }, "score": 0.748, "score_details": { "score": 0.748, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-nano/helm_capabilities_openai_gpt_5_nano_2025_08_07_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "google", "id": "google/gemini-2-5-pro-preview-03-25", "name": "Gemini 2.5 Pro 03-25 preview", }, "model_route_id": "google__gemini-2-5-pro-preview-03-25", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.745, }, }, "score": 0.745, "score_details": { "score": 0.745, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-pro-preview-03-25/helm_capabilities_google_gemini_2_5_pro_preview_03_25_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4", "name": "Claude 4 Sonnet 20250514", }, "model_route_id": "anthropic__claude-sonnet-4", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.733, }, }, "score": 0.733, "score_details": { "score": 0.733, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-sonnet-4/helm_capabilities_anthropic_claude_sonnet_4_20250514_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "xai", "id": "xai/grok-3-beta", "name": "Grok 3 Beta", }, "model_route_id": "xai__grok-3-beta", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.727, }, }, "score": 0.727, "score_details": { "score": 0.727, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-3-beta/helm_capabilities_xai_grok_3_beta_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-4-1", "name": "GPT-4.1 2025-04-14", }, "model_route_id": "openai__gpt-4-1", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.727, }, }, "score": 0.727, "score_details": { "score": 0.727, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1/helm_capabilities_openai_gpt_4_1_2025_04_14_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "qwen", "id": "qwen/qwen3-235b-a22b-fp8-tput", "name": "Qwen3 235B A22B FP8 Throughput", }, "model_route_id": "qwen__qwen3-235b-a22b-fp8-tput", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.726, }, }, "score": 0.726, "score_details": { "score": 0.726, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen3-235b-a22b-fp8-tput/helm_capabilities_qwen_qwen3_235b_a22b_fp8_tput_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-4-1-mini", "name": "GPT-4.1 mini 2025-04-14", }, "model_route_id": "openai__gpt-4-1-mini", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.726, }, }, "score": 0.726, "score_details": { "score": 0.726, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1-mini/helm_capabilities_openai_gpt_4_1_mini_2025_04_14_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "meta", "id": "meta/llama-4-maverick-17b-128e-instruct-fp8", "name": "Llama 4 Maverick 17Bx128E Instruct FP8", }, "model_route_id": "meta__llama-4-maverick-17b-128e-instruct-fp8", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.718, }, }, "score": 0.718, "score_details": { "score": 0.718, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-4-maverick-17b-128e-instruct-fp8/helm_capabilities_meta_llama_4_maverick_17b_128e_instruct_fp8_1774096304_5056698.json", }, { "evaluation_timestamp": "1776204744.2690232", "model_info": { "developer": "anthropic", "id": "anthropic/claude-haiku-4-5", "name": "Claude 4.5 Haiku 20251001", }, "model_route_id": "anthropic__claude-haiku-4-5", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1776204744.2690232", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.717, }, }, "score": 0.717, "score_details": { "score": 0.717, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-haiku-4-5/helm_capabilities_anthropic_claude_haiku_4_5_20251001_1776204744_2690232.json", }, { "evaluation_timestamp": "1776204744.2690232", "model_info": { "developer": "qwen", "id": "qwen/qwen3-next-80b-a3b-thinking", "name": "Qwen3-Next 80B A3B Thinking", }, "model_route_id": "qwen__qwen3-next-80b-a3b-thinking", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1776204744.2690232", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.7, }, }, "score": 0.7, "score_details": { "score": 0.7, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen3-next-80b-a3b-thinking/helm_capabilities_qwen_qwen3_next_80b_a3b_thinking_1776204744_2690232.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-r1-0528", "name": "DeepSeek-R1-0528", }, "model_route_id": "deepseek-ai__deepseek-r1-0528", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.699, }, }, "score": 0.699, "score_details": { "score": 0.699, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek-ai__deepseek-r1-0528/helm_capabilities_deepseek_ai_deepseek_r1_0528_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "writer", "id": "writer/palmyra-x5", "name": "Palmyra X5", }, "model_route_id": "writer__palmyra-x5", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.696, }, }, "score": 0.696, "score_details": { "score": 0.696, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-x5/helm_capabilities_writer_palmyra_x5_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "xai", "id": "xai/grok-3-mini-beta", "name": "Grok 3 mini Beta", }, "model_route_id": "xai__grok-3-mini-beta", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.679, }, }, "score": 0.679, "score_details": { "score": 0.679, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-3-mini-beta/helm_capabilities_xai_grok_3_mini_beta_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-001", "name": "Gemini 2.0 Flash", }, "model_route_id": "google__gemini-2-0-flash-001", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.679, }, }, "score": 0.679, "score_details": { "score": 0.679, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash-001/helm_capabilities_google_gemini_2_0_flash_001_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-oss-20b", "name": "gpt-oss-20b", }, "model_route_id": "openai__gpt-oss-20b", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.674, }, }, "score": 0.674, "score_details": { "score": 0.674, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-oss-20b/helm_capabilities_openai_gpt_oss_20b_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-7-sonnet", "name": "Claude 3.7 Sonnet 20250219", }, "model_route_id": "anthropic__claude-3-7-sonnet", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.674, }, }, "score": 0.674, "score_details": { "score": 0.674, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-7-sonnet/helm_capabilities_anthropic_claude_3_7_sonnet_20250219_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "zai-org", "id": "zai-org/glm-4-5-air-fp8", "name": "GLM-4.5-Air-FP8", }, "model_route_id": "zai-org__glm-4-5-air-fp8", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.67, }, }, "score": 0.67, "score_details": { "score": 0.67, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/zai-org__glm-4-5-air-fp8/helm_capabilities_zai_org_glm_4_5_air_fp8_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-v3", "name": "DeepSeek v3", }, "model_route_id": "deepseek-ai__deepseek-v3", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.665, }, }, "score": 0.665, "score_details": { "score": 0.665, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek-ai__deepseek-v3/helm_capabilities_deepseek_ai_deepseek_v3_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-002", "name": "Gemini 1.5 Pro 002", }, "model_route_id": "google__gemini-1-5-pro-002", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.657, }, }, "score": 0.657, "score_details": { "score": 0.657, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-pro-002/helm_capabilities_google_gemini_1_5_pro_002_1774096304_5056698.json", }, { "evaluation_timestamp": "1776204744.2690232", "model_info": { "developer": "openai", "id": "openai/gpt-5-1", "name": "GPT-5.1 2025-11-13", }, "model_route_id": "openai__gpt-5-1", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1776204744.2690232", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.656, }, }, "score": 0.656, "score_details": { "score": 0.656, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-1/helm_capabilities_openai_gpt_5_1_2025_11_13_1776204744_2690232.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-sonnet", "name": "Claude 3.5 Sonnet 20241022", }, "model_route_id": "anthropic__claude-3-5-sonnet", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.653, }, }, "score": 0.653, "score_details": { "score": 0.653, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-5-sonnet/helm_capabilities_anthropic_claude_3_5_sonnet_20241022_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "meta", "id": "meta/llama-4-scout-17b-16e-instruct", "name": "Llama 4 Scout 17Bx16E Instruct", }, "model_route_id": "meta__llama-4-scout-17b-16e-instruct", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.644, }, }, "score": 0.644, "score_details": { "score": 0.644, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-4-scout-17b-16e-instruct/helm_capabilities_meta_llama_4_scout_17b_16e_instruct_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-lite-preview-02-05", "name": "Gemini 2.0 Flash Lite 02-05 preview", }, "model_route_id": "google__gemini-2-0-flash-lite-preview-02-05", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.642, }, }, "score": 0.642, "score_details": { "score": 0.642, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash-lite-preview-02-05/helm_capabilities_google_gemini_2_0_flash_lite_preview_02_05_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "amazon", "id": "amazon/nova-premier-v1-0", "name": "Amazon Nova Premier", }, "model_route_id": "amazon__nova-premier-v1-0", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.637, }, }, "score": 0.637, "score_details": { "score": 0.637, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/amazon__nova-premier-v1-0/helm_capabilities_amazon_nova_premier_v1_0_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o 2024-11-20", }, "model_route_id": "openai__gpt-4o", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.634, }, }, "score": 0.634, "score_details": { "score": 0.634, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o/helm_capabilities_openai_gpt_4o_2024_11_20_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-preview-04-17", "name": "Gemini 2.5 Flash 04-17 preview", }, "model_route_id": "google__gemini-2-5-flash-preview-04-17", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.626, }, }, "score": 0.626, "score_details": { "score": 0.626, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash-preview-04-17/helm_capabilities_google_gemini_2_5_flash_preview_04_17_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "meta", "id": "meta/llama-3-1-405b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 405B", }, "model_route_id": "meta__llama-3-1-405b-instruct-turbo", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.618, }, }, "score": 0.618, "score_details": { "score": 0.618, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-405b-instruct-turbo/helm_capabilities_meta_llama_3_1_405b_instruct_turbo_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-4-1-nano", "name": "GPT-4.1 nano 2025-04-14", }, "model_route_id": "openai__gpt-4-1-nano", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.616, }, }, "score": 0.616, "score_details": { "score": 0.616, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1-nano/helm_capabilities_openai_gpt_4_1_nano_2025_04_14_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "writer", "id": "writer/palmyra-x-004", "name": "Palmyra-X-004", }, "model_route_id": "writer__palmyra-x-004", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.609, }, }, "score": 0.609, "score_details": { "score": 0.609, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-x-004/helm_capabilities_writer_palmyra_x_004_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-002", "name": "Gemini 1.5 Flash 002", }, "model_route_id": "google__gemini-1-5-flash-002", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.609, }, }, "score": 0.609, "score_details": { "score": 0.609, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-flash-002/helm_capabilities_google_gemini_1_5_flash_002_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-72b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 72B", }, "model_route_id": "qwen__qwen2-5-72b-instruct-turbo", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.599, }, }, "score": 0.599, "score_details": { "score": 0.599, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen2-5-72b-instruct-turbo/helm_capabilities_qwen_qwen2_5_72b_instruct_turbo_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-large-2411", "name": "Mistral Large 2411", }, "model_route_id": "mistralai__mistral-large-2411", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.598, }, }, "score": 0.598, "score_details": { "score": 0.598, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-large-2411/helm_capabilities_mistralai_mistral_large_2411_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-lite", "name": "Gemini 2.5 Flash-Lite", }, "model_route_id": "google__gemini-2-5-flash-lite", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.591, }, }, "score": 0.591, "score_details": { "score": 0.591, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash-lite/helm_capabilities_google_gemini_2_5_flash_lite_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "amazon", "id": "amazon/nova-pro-v1-0", "name": "Amazon Nova Pro", }, "model_route_id": "amazon__nova-pro-v1-0", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.591, }, }, "score": 0.591, "score_details": { "score": 0.591, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/amazon__nova-pro-v1-0/helm_capabilities_amazon_nova_pro_v1_0_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "writer", "id": "writer/palmyra-fin", "name": "Palmyra Fin", }, "model_route_id": "writer__palmyra-fin", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.577, }, }, "score": 0.577, "score_details": { "score": 0.577, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-fin/helm_capabilities_writer_palmyra_fin_1774096304_5056698.json", }, { "evaluation_timestamp": "1776204744.2690232", "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-h-small", "name": "IBM Granite 4.0 Small", }, "model_route_id": "ibm__granite-4-0-h-small", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1776204744.2690232", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.575, }, }, "score": 0.575, "score_details": { "score": 0.575, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ibm__granite-4-0-h-small/helm_capabilities_ibm_granite_4_0_h_small_1776204744_2690232.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "meta", "id": "meta/llama-3-1-70b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 70B", }, "model_route_id": "meta__llama-3-1-70b-instruct-turbo", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.574, }, }, "score": 0.574, "score_details": { "score": 0.574, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-70b-instruct-turbo/helm_capabilities_meta_llama_3_1_70b_instruct_turbo_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "openai", "id": "openai/gpt-4o-mini", "name": "GPT-4o mini 2024-07-18", }, "model_route_id": "openai__gpt-4o-mini", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.565, }, }, "score": 0.565, "score_details": { "score": 0.565, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o-mini/helm_capabilities_openai_gpt_4o_mini_2024_07_18_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-small-2503", "name": "Mistral Small 3.1 2503", }, "model_route_id": "mistralai__mistral-small-2503", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.558, }, }, "score": 0.558, "score_details": { "score": 0.558, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-small-2503/helm_capabilities_mistralai_mistral_small_2503_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "amazon", "id": "amazon/nova-lite-v1-0", "name": "Amazon Nova Lite", }, "model_route_id": "amazon__nova-lite-v1-0", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.551, }, }, "score": 0.551, "score_details": { "score": 0.551, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/amazon__nova-lite-v1-0/helm_capabilities_amazon_nova_lite_v1_0_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-haiku", "name": "Claude 3.5 Haiku 20241022", }, "model_route_id": "anthropic__claude-3-5-haiku", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.549, }, }, "score": 0.549, "score_details": { "score": 0.549, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-5-haiku/helm_capabilities_anthropic_claude_3_5_haiku_20241022_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-7b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 7B", }, "model_route_id": "qwen__qwen2-5-7b-instruct-turbo", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.529, }, }, "score": 0.529, "score_details": { "score": 0.529, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen2-5-7b-instruct-turbo/helm_capabilities_qwen_qwen2_5_7b_instruct_turbo_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "amazon", "id": "amazon/nova-micro-v1-0", "name": "Amazon Nova Micro", }, "model_route_id": "amazon__nova-micro-v1-0", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.522, }, }, "score": 0.522, "score_details": { "score": 0.522, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/amazon__nova-micro-v1-0/helm_capabilities_amazon_nova_micro_v1_0_1774096304_5056698.json", }, { "evaluation_timestamp": "1776204744.2690232", "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-micro", "name": "IBM Granite 4.0 Micro", }, "model_route_id": "ibm__granite-4-0-micro", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1776204744.2690232", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.486, }, }, "score": 0.486, "score_details": { "score": 0.486, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ibm__granite-4-0-micro/helm_capabilities_ibm_granite_4_0_micro_1776204744_2690232.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x22b-instruct-v0-1", "name": "Mixtral Instruct 8x22B", }, "model_route_id": "mistralai__mixtral-8x22b-instruct-v0-1", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.478, }, }, "score": 0.478, "score_details": { "score": 0.478, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mixtral-8x22b-instruct-v0-1/helm_capabilities_mistralai_mixtral_8x22b_instruct_v0_1_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "writer", "id": "writer/palmyra-med", "name": "Palmyra Med", }, "model_route_id": "writer__palmyra-med", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.476, }, }, "score": 0.476, "score_details": { "score": 0.476, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-med/helm_capabilities_writer_palmyra_med_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "allenai", "id": "allenai/olmo-2-0325-32b-instruct", "name": "OLMo 2 32B Instruct March 2025", }, "model_route_id": "allenai__olmo-2-0325-32b-instruct", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.475, }, }, "score": 0.475, "score_details": { "score": 0.475, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/allenai__olmo-2-0325-32b-instruct/helm_capabilities_allenai_olmo_2_0325_32b_instruct_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "ibm", "id": "ibm/granite-3-3-8b-instruct", "name": "IBM Granite 3.3 8B Instruct", }, "model_route_id": "ibm__granite-3-3-8b-instruct", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.463, }, }, "score": 0.463, "score_details": { "score": 0.463, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ibm__granite-3-3-8b-instruct/helm_capabilities_ibm_granite_3_3_8b_instruct_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "meta", "id": "meta/llama-3-1-8b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 8B", }, "model_route_id": "meta__llama-3-1-8b-instruct-turbo", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.444, }, }, "score": 0.444, "score_details": { "score": 0.444, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-8b-instruct-turbo/helm_capabilities_meta_llama_3_1_8b_instruct_turbo_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "allenai", "id": "allenai/olmo-2-1124-13b-instruct", "name": "OLMo 2 13B Instruct November 2024", }, "model_route_id": "allenai__olmo-2-1124-13b-instruct", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.44, }, }, "score": 0.44, "score_details": { "score": 0.44, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/allenai__olmo-2-1124-13b-instruct/helm_capabilities_allenai_olmo_2_1124_13b_instruct_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "allenai", "id": "allenai/olmo-2-1124-7b-instruct", "name": "OLMo 2 7B Instruct November 2024", }, "model_route_id": "allenai__olmo-2-1124-7b-instruct", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.405, }, }, "score": 0.405, "score_details": { "score": 0.405, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/allenai__olmo-2-1124-7b-instruct/helm_capabilities_allenai_olmo_2_1124_7b_instruct_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x7b-instruct-v0-1", "name": "Mixtral Instruct 8x7B", }, "model_route_id": "mistralai__mixtral-8x7b-instruct-v0-1", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.397, }, }, "score": 0.397, "score_details": { "score": 0.397, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mixtral-8x7b-instruct-v0-1/helm_capabilities_mistralai_mixtral_8x7b_instruct_v0_1_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-instruct-v0-3", "name": "Mistral Instruct v0.3 7B", }, "model_route_id": "mistralai__mistral-7b-instruct-v0-3", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.376, }, }, "score": 0.376, "score_details": { "score": 0.376, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-7b-instruct-v0-3/helm_capabilities_mistralai_mistral_7b_instruct_v0_3_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "allenai", "id": "allenai/olmoe-1b-7b-0125-instruct", "name": "OLMoE 1B-7B Instruct January 2025", }, "model_route_id": "allenai__olmoe-1b-7b-0125-instruct", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.332, }, }, "score": 0.332, "score_details": { "score": 0.332, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/allenai__olmoe-1b-7b-0125-instruct/helm_capabilities_allenai_olmoe_1b_7b_0125_instruct_1774096304_5056698.json", }, { "evaluation_timestamp": "1774096304.5056698", "model_info": { "developer": "marin-community", "id": "marin-community/marin-8b-instruct", "name": "Marin 8B Instruct", }, "model_route_id": "marin-community__marin-8b-instruct", "result": { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "detailed_evaluation_results_url": undefined, "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Score", "evaluation_timestamp": "1774096304.5056698", "metric_config": { "evaluation_description": "The mean of the scores from all columns.", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "score", "metric_summary_id": "helm_capabilities_score", "score_details": { "score": 0.325, }, }, "score": 0.325, "score_details": { "score": 0.325, }, "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_capabilities", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/marin-community__marin-8b-instruct/helm_capabilities_marin_community_marin_8b_instruct_1774096304_5056698.json", }, ], "models_count": 68, "provenance_summary": { "first_party_only_groups": 0, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 0, "third_party": 68, "unspecified": 0, }, "total_groups": 66, "total_results": 68, }, "reproducibility_summary": { "has_reproducibility_gap_count": 68, "populated_ratio_avg": 0, "results_total": 68, }, "root_metrics": [ { "canonical_display_name": "Holistic Evaluation of Language Models (HELM) / Score", "display_name": "Holistic Evaluation of Language Models (HELM) / Score", "lower_is_better": false, "metric_key": "score", "metric_name": "Score", "metric_summary_id": "helm_capabilities_score", "models_count": 68, "top_score": 0.819, "unit": "proportion", }, ], "source_data": { "dataset_name": "helm_capabilities", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json", ], }, "source_types": [], "subtasks": [], "third_party_ratio": 0, "worst_model": { "name": "Marin 8B Instruct", "score": 0.325, }, } `; exports[`hfEvalDetailToSummary > helm_classic_truthfulqa — Safety regression-bait — pipeline category=knowledge but inferCategoryFromBenchmark returns Safety 1`] = ` { "avg_score": 0.2772686567164178, "avg_score_norm": 0.2772686567164178, "benchmark_card": { "benchmark_details": { "benchmark_type": "composite", "contains": [ "BoolQ", "CNN/DailyMail", "CivilComments", "HellaSwag", "IMDB", "MMLU", "MS MARCO (TREC)", "NarrativeQA", "NaturalQuestions (open-book)", "OpenbookQA", "QuAC", "RAFT", "TruthfulQA", "XSUM", ], "data_type": "text", "domains": [ "question answering", "summarization", "reasoning", "bias detection", "language understanding", "language generation", ], "languages": [ "English", ], "name": "helm_classic", "overview": "helm_classic is a composite benchmark suite designed for holistic evaluation of language models across multiple capabilities and domains. It aggregates several established benchmarks including BoolQ, CNN/DailyMail, CivilComments, HellaSwag, IMDB, MMLU, MS MARCO (TREC), NarrativeQA, NaturalQuestions (open-book), OpenbookQA, QuAC, RAFT, TruthfulQA, and XSUM. The suite measures a wide range of language understanding and generation tasks such as question answering, summarization, reasoning, and bias detection, providing comprehensive coverage of model performance in diverse scenarios.", "resources": [ "https://crfm.stanford.edu/helm/classic/latest/", "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", "https://arxiv.org/abs/2211.09110", ], "similar_benchmarks": [ "GEM", "XTREME", "GEMv2", ], }, "data": { "annotation": "Not specified", "format": "Not specified", "size": "Not specified", "source": "The benchmark suite aggregates data from multiple existing datasets: BoolQ, CNN/DailyMail, CivilComments, HellaSwag, IMDB, MMLU, MS MARCO (TREC), NarrativeQA, NaturalQuestions (open-book), OpenbookQA, QuAC, RAFT, TruthfulQA, and XSUM.", }, "ethical_and_legal_considerations": { "compliance_with_regulations": "Not specified", "consent_procedures": "Not specified", "data_licensing": "Not specified", "privacy_and_anonymity": "Not specified", }, "flagged_fields": {}, "methodology": { "baseline_results": "Not specified", "calculation": "Metrics are calculated per sub-benchmark using standard evaluation protocols: EM for exact answer matching, ROUGE-2 for summarization quality, NDCG@10 for ranking tasks, and F1 for token-level overlap", "interpretation": "Higher scores indicate better performance across all metrics", "methods": [ "The benchmark evaluates language models across multiple tasks including question answering, summarization, reasoning, and language understanding", "It employs a holistic evaluation framework that aggregates performance from diverse sub-benchmarks", ], "metrics": [ "Exact Match (EM)", "ROUGE-2", "Normalized Discounted Cumulative Gain (NDCG@10)", "F1 score", ], "validation": "Not specified", }, "missing_fields": [], "possible_risks": [ { "category": "Over- or under-reliance", "description": [ "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html", }, { "category": "Unrepresentative data", "description": [ "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html", }, { "category": "Uncertain data provenance", "description": [ "Data provenance refers to the traceability of data (including synthetic data), which includes its ownership, origin, transformations, and generation. Proving that the data is the same as the original source with correct usage terms is difficult without standardized methods for verifying data sources or generation.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-provenance.html", }, { "category": "Data contamination", "description": [ "Data contamination occurs when incorrect data is used for training. For example, data that is not aligned with model's purpose or data that is already set aside for other development tasks such as testing and evaluation.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-contamination.html", }, { "category": "Reproducibility", "description": [ "Replicating agent behavior or output can be impacted by changes or updates made to external services and tools. This impact is increased if the agent is built with generative AI.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/reproducibility-agentic.html", }, ], "purpose_and_intended_users": { "audience": [ "NLP researchers", "Language model developers", ], "goal": "To holistically evaluate language models across a comprehensive set of tasks and domains, including question answering, summarization, reasoning, and classification, by aggregating performance from multiple established benchmarks.", "limitations": "Not specified", "out_of_scope_uses": [ "Not specified", ], "tasks": [ "Question answering", "Text summarization", "Text classification", "Reasoning", "Toxic comment identification", ], }, }, "best_model": { "name": "Palmyra X 43B", "score": 0.616, }, "canonical_display_name": "TruthfulQA", "category": "Safety", "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 0, "total_groups": 67, "variant_divergent_count": 0, }, "composite_benchmark_key": "TruthfulQA", "composite_benchmark_name": "TruthfulQA", "evalcards": { "annotations": { "benchmark_comparability": { "cross_party_divergence_groups": [], "variant_divergence_groups": [], }, "reporting_completeness": { "completeness_score": 0.9285714285714286, "field_scores": [ { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.name", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.overview", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.data_type", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.domains", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.languages", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.similar_benchmarks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.resources", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.goal", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.audience", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.tasks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.limitations", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.methods", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.metrics", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.calculation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.interpretation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.baseline_results", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.validation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "score": 1, }, { "coverage_type": "partial", "field_path": "autobenchmarkcard.data", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_type", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_organization_name", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.evaluator_relationship", "score": 1, }, { "coverage_type": "reserved", "field_path": "evalcards.lifecycle_status", "score": 0, }, { "coverage_type": "reserved", "field_path": "evalcards.preregistration_url", "score": 0, }, ], "missing_required_fields": [ "evalcards.lifecycle_status", "evalcards.preregistration_url", ], "partial_fields": [], "signal_version": "1.0", "total_fields_evaluated": 28, }, }, }, "evaluation_id": "helm_classic_truthfulqa", "evaluation_name": "TruthfulQA", "evaluator_names": [], "latest_source_name": "TruthfulQA", "leaderboard_metrics": [ { "canonical_display_name": "TruthfulQA / Exact Match", "column_key": "root:helm_classic_truthfulqa_exact_match", "display_name": "TruthfulQA / Exact Match", "lower_is_better": false, "metric_name": "Exact Match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "scope": "root", "subtask_key": undefined, "subtask_name": undefined, "unit": "proportion", }, ], "leaderboard_rows": [ { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/palmyra-x-43b", "name": "Palmyra X 43B", }, "model_route_id": "google__palmyra-x-43b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.616, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/text-davinci-002", "name": "text-davinci-002", }, "model_route_id": "openai__text-davinci-002", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.61, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0301", "name": "gpt-3.5-turbo-0301", }, "model_route_id": "openai__gpt-3-5-turbo-0301", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.609, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/text-davinci-003", "name": "text-davinci-003", }, "model_route_id": "openai__text-davinci-003", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.593, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-2-70b", "name": "Llama 2 70B", }, "model_route_id": "meta__llama-2-70b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.554, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-65b", "name": "LLaMA 65B", }, "model_route_id": "meta__llama-65b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.508, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/jurassic-2-jumbo-178b", "name": "Jurassic-2 Jumbo 178B", }, "model_route_id": "ai21__jurassic-2-jumbo-178b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.437, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-v0-1-7b", "name": "Mistral v0.1 7B", }, "model_route_id": "mistralai__mistral-v0-1-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.422, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "lmsys", "id": "lmsys/vicuna-v1-3-13b", "name": "Vicuna v1.3 13B", }, "model_route_id": "lmsys__vicuna-v1-3-13b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.385, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-instruct-40b", "name": "Falcon-Instruct 40B", }, "model_route_id": "tiiuae__falcon-instruct-40b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.384, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "bigscience", "id": "bigscience/t0pp-11b", "name": "T0pp 11B", }, "model_route_id": "bigscience__t0pp-11b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.377, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "unknown", "id": "unknown/anthropic-lm-v4-s3-52b", "name": "Anthropic-LM v4-s3 52B", }, "model_route_id": "unknown__anthropic-lm-v4-s3-52b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.368, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-40b", "name": "Falcon 40B", }, "model_route_id": "tiiuae__falcon-40b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.353, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/jurassic-2-grande-17b", "name": "Jurassic-2 Grande 17B", }, "model_route_id": "ai21__jurassic-2-grande-17b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.348, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-30b", "name": "LLaMA 30B", }, "model_route_id": "meta__llama-30b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.344, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0613", "name": "gpt-3.5-turbo-0613", }, "model_route_id": "openai__gpt-3-5-turbo-0613", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.339, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-2-13b", "name": "Llama 2 13B", }, "model_route_id": "meta__llama-2-13b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.33, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-13b", "name": "LLaMA 13B", }, "model_route_id": "meta__llama-13b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.324, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/j1-grande-v2-beta-17b", "name": "J1-Grande v2 beta 17B", }, "model_route_id": "ai21__j1-grande-v2-beta-17b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.306, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "lmsys", "id": "lmsys/vicuna-v1-3-7b", "name": "Vicuna v1.3 7B", }, "model_route_id": "lmsys__vicuna-v1-3-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.292, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-7b", "name": "LLaMA 7B", }, "model_route_id": "meta__llama-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.28, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "together", "id": "together/redpajama-incite-base-v1-3b", "name": "RedPajama-INCITE-Base-v1 3B", }, "model_route_id": "together__redpajama-incite-base-v1-3b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.277, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-2-7b", "name": "Llama 2 7B", }, "model_route_id": "meta__llama-2-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.272, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/cohere-command-beta-52-4b", "name": "Cohere Command beta 52.4B", }, "model_route_id": "cohere__cohere-command-beta-52-4b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.269, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/text-curie-001", "name": "text-curie-001", }, "model_route_id": "openai__text-curie-001", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.257, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "microsoft", "id": "microsoft/tnlg-v2-530b", "name": "TNLG v2 530B", }, "model_route_id": "microsoft__tnlg-v2-530b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.251, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/opt-175b", "name": "OPT 175B", }, "model_route_id": "meta__opt-175b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.25, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/jurassic-2-large-7-5b", "name": "Jurassic-2 Large 7.5B", }, "model_route_id": "ai21__jurassic-2-large-7-5b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.245, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "together", "id": "together/redpajama-incite-instruct-7b", "name": "RedPajama-INCITE-Instruct 7B", }, "model_route_id": "together__redpajama-incite-instruct-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.243, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "stanford", "id": "stanford/alpaca-7b", "name": "Alpaca 7B", }, "model_route_id": "stanford__alpaca-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.243, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-7b", "name": "Falcon 7B", }, "model_route_id": "tiiuae__falcon-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.234, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "mosaicml", "id": "mosaicml/mpt-instruct-30b", "name": "MPT-Instruct 30B", }, "model_route_id": "mosaicml__mpt-instruct-30b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.234, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/text-babbage-001", "name": "text-babbage-001", }, "model_route_id": "openai__text-babbage-001", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.233, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/text-ada-001", "name": "text-ada-001", }, "model_route_id": "openai__text-ada-001", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.232, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/curie-6-7b", "name": "curie 6.7B", }, "model_route_id": "openai__curie-6-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.232, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "mosaicml", "id": "mosaicml/mpt-30b", "name": "MPT 30B", }, "model_route_id": "mosaicml__mpt-30b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.231, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "aleph-alpha", "id": "aleph-alpha/luminous-supreme-70b", "name": "Luminous Supreme 70B", }, "model_route_id": "aleph-alpha__luminous-supreme-70b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.222, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "aleph-alpha", "id": "aleph-alpha/luminous-extended-30b", "name": "Luminous Extended 30B", }, "model_route_id": "aleph-alpha__luminous-extended-30b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.221, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "zhipu-ai", "id": "zhipu-ai/glm-130b", "name": "GLM 130B", }, "model_route_id": "zhipu-ai__glm-130b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.218, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/cohere-small-v20220720-410m", "name": "Cohere small v20220720 410M", }, "model_route_id": "cohere__cohere-small-v20220720-410m", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.217, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-neox-20b", "name": "GPT-NeoX 20B", }, "model_route_id": "openai__gpt-neox-20b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.216, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/ada-350m", "name": "ada 350M", }, "model_route_id": "openai__ada-350m", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.215, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/cohere-medium-v20221108-6-1b", "name": "Cohere medium v20221108 6.1B", }, "model_route_id": "cohere__cohere-medium-v20221108-6-1b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.215, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-instruct-7b", "name": "Falcon-Instruct 7B", }, "model_route_id": "tiiuae__falcon-instruct-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.213, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "eleutherai", "id": "eleutherai/pythia-6-9b", "name": "Pythia 6.9B", }, "model_route_id": "eleutherai__pythia-6-9b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.213, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "together", "id": "together/redpajama-incite-instruct-v1-3b", "name": "RedPajama-INCITE-Instruct-v1 3B", }, "model_route_id": "together__redpajama-incite-instruct-v1-3b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.208, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "together", "id": "together/redpajama-incite-base-7b", "name": "RedPajama-INCITE-Base 7B", }, "model_route_id": "together__redpajama-incite-base-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.205, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "bigscience", "id": "bigscience/bloom-176b", "name": "BLOOM 176B", }, "model_route_id": "bigscience__bloom-176b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.205, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/cohere-command-beta-6-1b", "name": "Cohere Command beta 6.1B", }, "model_route_id": "cohere__cohere-command-beta-6-1b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.203, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "yandex", "id": "yandex/yalm-100b", "name": "YaLM 100B", }, "model_route_id": "yandex__yalm-100b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.202, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/opt-66b", "name": "OPT 66B", }, "model_route_id": "meta__opt-66b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.201, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-j-6b", "name": "GPT-J 6B", }, "model_route_id": "openai__gpt-j-6b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.199, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/cohere-xlarge-v20220609-52-4b", "name": "Cohere xlarge v20220609 52.4B", }, "model_route_id": "cohere__cohere-xlarge-v20220609-52-4b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.198, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/j1-large-v1-7-5b", "name": "J1-Large v1 7.5B", }, "model_route_id": "ai21__j1-large-v1-7-5b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.197, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/davinci-175b", "name": "davinci 175B", }, "model_route_id": "openai__davinci-175b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.194, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/ul2-20b", "name": "UL2 20B", }, "model_route_id": "google__ul2-20b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.193, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/j1-grande-v1-17b", "name": "J1-Grande v1 17B", }, "model_route_id": "ai21__j1-grande-v1-17b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.193, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/cohere-medium-v20220720-6-1b", "name": "Cohere medium v20220720 6.1B", }, "model_route_id": "cohere__cohere-medium-v20220720-6-1b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.19, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/babbage-1-3b", "name": "babbage 1.3B", }, "model_route_id": "openai__babbage-1-3b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.188, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/instructpalmyra-30b", "name": "InstructPalmyra 30B", }, "model_route_id": "writer__instructpalmyra-30b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.185, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "aleph-alpha", "id": "aleph-alpha/luminous-base-13b", "name": "Luminous Base 13B", }, "model_route_id": "aleph-alpha__luminous-base-13b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.182, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/cohere-large-v20220720-13-1b", "name": "Cohere large v20220720 13.1B", }, "model_route_id": "cohere__cohere-large-v20220720-13-1b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.181, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "eleutherai", "id": "eleutherai/pythia-12b", "name": "Pythia 12B", }, "model_route_id": "eleutherai__pythia-12b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.177, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/j1-jumbo-v1-178b", "name": "J1-Jumbo v1 178B", }, "model_route_id": "ai21__j1-jumbo-v1-178b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.175, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/cohere-xlarge-v20221108-52-4b", "name": "Cohere xlarge v20221108 52.4B", }, "model_route_id": "cohere__cohere-xlarge-v20221108-52-4b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.169, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "microsoft", "id": "microsoft/tnlg-v2-6-7b", "name": "TNLG v2 6.7B", }, "model_route_id": "microsoft__tnlg-v2-6-7b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.167, }, }, { "annotations_by_metric": { "root:helm_classic_truthfulqa_exact_match": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096308.339228", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/t5-11b", "name": "T5 11B", }, "model_route_id": "google__t5-11b", "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_classic_truthfulqa_exact_match": 0.133, }, }, ], "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_names": [ "Exact Match", ], "metrics_count": 1, "missing_generation_config_count": 0, "model_results": [ { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "google", "id": "google/palmyra-x-43b", "name": "Palmyra X 43B", }, "model_route_id": "google__palmyra-x-43b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.616, }, }, "score": 0.616, "score_details": { "score": 0.616, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__palmyra-x-43b/helm_classic_google_palmyra_x_43b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/text-davinci-002", "name": "text-davinci-002", }, "model_route_id": "openai__text-davinci-002", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.61, }, }, "score": 0.61, "score_details": { "score": 0.61, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__text-davinci-002/helm_classic_openai_text_davinci_002_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0301", "name": "gpt-3.5-turbo-0301", }, "model_route_id": "openai__gpt-3-5-turbo-0301", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.609, }, }, "score": 0.609, "score_details": { "score": 0.609, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-3-5-turbo-0301/helm_classic_openai_gpt_3_5_turbo_0301_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/text-davinci-003", "name": "text-davinci-003", }, "model_route_id": "openai__text-davinci-003", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.593, }, }, "score": 0.593, "score_details": { "score": 0.593, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__text-davinci-003/helm_classic_openai_text_davinci_003_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "meta", "id": "meta/llama-2-70b", "name": "Llama 2 70B", }, "model_route_id": "meta__llama-2-70b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.554, }, }, "score": 0.554, "score_details": { "score": 0.554, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-2-70b/helm_classic_meta_llama_2_70b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "meta", "id": "meta/llama-65b", "name": "LLaMA 65B", }, "model_route_id": "meta__llama-65b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.508, }, }, "score": 0.508, "score_details": { "score": 0.508, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-65b/helm_classic_meta_llama_65b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "ai21", "id": "ai21/jurassic-2-jumbo-178b", "name": "Jurassic-2 Jumbo 178B", }, "model_route_id": "ai21__jurassic-2-jumbo-178b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.437, }, }, "score": 0.437, "score_details": { "score": 0.437, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__jurassic-2-jumbo-178b/helm_classic_ai21_jurassic_2_jumbo_178b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-v0-1-7b", "name": "Mistral v0.1 7B", }, "model_route_id": "mistralai__mistral-v0-1-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.422, }, }, "score": 0.422, "score_details": { "score": 0.422, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-v0-1-7b/helm_classic_mistralai_mistral_v0_1_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "lmsys", "id": "lmsys/vicuna-v1-3-13b", "name": "Vicuna v1.3 13B", }, "model_route_id": "lmsys__vicuna-v1-3-13b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.385, }, }, "score": 0.385, "score_details": { "score": 0.385, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/lmsys__vicuna-v1-3-13b/helm_classic_lmsys_vicuna_v1_3_13b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-instruct-40b", "name": "Falcon-Instruct 40B", }, "model_route_id": "tiiuae__falcon-instruct-40b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.384, }, }, "score": 0.384, "score_details": { "score": 0.384, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/tiiuae__falcon-instruct-40b/helm_classic_tiiuae_falcon_instruct_40b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "bigscience", "id": "bigscience/t0pp-11b", "name": "T0pp 11B", }, "model_route_id": "bigscience__t0pp-11b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.377, }, }, "score": 0.377, "score_details": { "score": 0.377, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/bigscience__t0pp-11b/helm_classic_bigscience_t0pp_11b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "unknown", "id": "unknown/anthropic-lm-v4-s3-52b", "name": "Anthropic-LM v4-s3 52B", }, "model_route_id": "unknown__anthropic-lm-v4-s3-52b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.368, }, }, "score": 0.368, "score_details": { "score": 0.368, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/unknown__anthropic-lm-v4-s3-52b/helm_classic_anthropic_lm_v4_s3_52b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-40b", "name": "Falcon 40B", }, "model_route_id": "tiiuae__falcon-40b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.353, }, }, "score": 0.353, "score_details": { "score": 0.353, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/tiiuae__falcon-40b/helm_classic_tiiuae_falcon_40b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "ai21", "id": "ai21/jurassic-2-grande-17b", "name": "Jurassic-2 Grande 17B", }, "model_route_id": "ai21__jurassic-2-grande-17b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.348, }, }, "score": 0.348, "score_details": { "score": 0.348, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__jurassic-2-grande-17b/helm_classic_ai21_jurassic_2_grande_17b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "meta", "id": "meta/llama-30b", "name": "LLaMA 30B", }, "model_route_id": "meta__llama-30b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.344, }, }, "score": 0.344, "score_details": { "score": 0.344, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-30b/helm_classic_meta_llama_30b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0613", "name": "gpt-3.5-turbo-0613", }, "model_route_id": "openai__gpt-3-5-turbo-0613", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.339, }, }, "score": 0.339, "score_details": { "score": 0.339, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-3-5-turbo-0613/helm_classic_openai_gpt_3_5_turbo_0613_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "meta", "id": "meta/llama-2-13b", "name": "Llama 2 13B", }, "model_route_id": "meta__llama-2-13b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.33, }, }, "score": 0.33, "score_details": { "score": 0.33, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-2-13b/helm_classic_meta_llama_2_13b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "meta", "id": "meta/llama-13b", "name": "LLaMA 13B", }, "model_route_id": "meta__llama-13b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.324, }, }, "score": 0.324, "score_details": { "score": 0.324, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-13b/helm_classic_meta_llama_13b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "ai21", "id": "ai21/j1-grande-v2-beta-17b", "name": "J1-Grande v2 beta 17B", }, "model_route_id": "ai21__j1-grande-v2-beta-17b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.306, }, }, "score": 0.306, "score_details": { "score": 0.306, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__j1-grande-v2-beta-17b/helm_classic_ai21_j1_grande_v2_beta_17b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "lmsys", "id": "lmsys/vicuna-v1-3-7b", "name": "Vicuna v1.3 7B", }, "model_route_id": "lmsys__vicuna-v1-3-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.292, }, }, "score": 0.292, "score_details": { "score": 0.292, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/lmsys__vicuna-v1-3-7b/helm_classic_lmsys_vicuna_v1_3_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "meta", "id": "meta/llama-7b", "name": "LLaMA 7B", }, "model_route_id": "meta__llama-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.28, }, }, "score": 0.28, "score_details": { "score": 0.28, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-7b/helm_classic_meta_llama_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "together", "id": "together/redpajama-incite-base-v1-3b", "name": "RedPajama-INCITE-Base-v1 3B", }, "model_route_id": "together__redpajama-incite-base-v1-3b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.277, }, }, "score": 0.277, "score_details": { "score": 0.277, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/together__redpajama-incite-base-v1-3b/helm_classic_together_redpajama_incite_base_v1_3b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "meta", "id": "meta/llama-2-7b", "name": "Llama 2 7B", }, "model_route_id": "meta__llama-2-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.272, }, }, "score": 0.272, "score_details": { "score": 0.272, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-2-7b/helm_classic_meta_llama_2_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "cohere", "id": "cohere/cohere-command-beta-52-4b", "name": "Cohere Command beta 52.4B", }, "model_route_id": "cohere__cohere-command-beta-52-4b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.269, }, }, "score": 0.269, "score_details": { "score": 0.269, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__cohere-command-beta-52-4b/helm_classic_cohere_cohere_command_beta_52_4b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/text-curie-001", "name": "text-curie-001", }, "model_route_id": "openai__text-curie-001", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.257, }, }, "score": 0.257, "score_details": { "score": 0.257, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__text-curie-001/helm_classic_openai_text_curie_001_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "microsoft", "id": "microsoft/tnlg-v2-530b", "name": "TNLG v2 530B", }, "model_route_id": "microsoft__tnlg-v2-530b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.251, }, }, "score": 0.251, "score_details": { "score": 0.251, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/microsoft__tnlg-v2-530b/helm_classic_microsoft_tnlg_v2_530b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "meta", "id": "meta/opt-175b", "name": "OPT 175B", }, "model_route_id": "meta__opt-175b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.25, }, }, "score": 0.25, "score_details": { "score": 0.25, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__opt-175b/helm_classic_meta_opt_175b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "ai21", "id": "ai21/jurassic-2-large-7-5b", "name": "Jurassic-2 Large 7.5B", }, "model_route_id": "ai21__jurassic-2-large-7-5b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.245, }, }, "score": 0.245, "score_details": { "score": 0.245, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__jurassic-2-large-7-5b/helm_classic_ai21_jurassic_2_large_7_5b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "together", "id": "together/redpajama-incite-instruct-7b", "name": "RedPajama-INCITE-Instruct 7B", }, "model_route_id": "together__redpajama-incite-instruct-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.243, }, }, "score": 0.243, "score_details": { "score": 0.243, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/together__redpajama-incite-instruct-7b/helm_classic_together_redpajama_incite_instruct_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "stanford", "id": "stanford/alpaca-7b", "name": "Alpaca 7B", }, "model_route_id": "stanford__alpaca-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.243, }, }, "score": 0.243, "score_details": { "score": 0.243, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/stanford__alpaca-7b/helm_classic_stanford_alpaca_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-7b", "name": "Falcon 7B", }, "model_route_id": "tiiuae__falcon-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.234, }, }, "score": 0.234, "score_details": { "score": 0.234, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/tiiuae__falcon-7b/helm_classic_tiiuae_falcon_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "mosaicml", "id": "mosaicml/mpt-instruct-30b", "name": "MPT-Instruct 30B", }, "model_route_id": "mosaicml__mpt-instruct-30b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.234, }, }, "score": 0.234, "score_details": { "score": 0.234, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mosaicml__mpt-instruct-30b/helm_classic_mosaicml_mpt_instruct_30b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/text-babbage-001", "name": "text-babbage-001", }, "model_route_id": "openai__text-babbage-001", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.233, }, }, "score": 0.233, "score_details": { "score": 0.233, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__text-babbage-001/helm_classic_openai_text_babbage_001_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/text-ada-001", "name": "text-ada-001", }, "model_route_id": "openai__text-ada-001", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.232, }, }, "score": 0.232, "score_details": { "score": 0.232, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__text-ada-001/helm_classic_openai_text_ada_001_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/curie-6-7b", "name": "curie 6.7B", }, "model_route_id": "openai__curie-6-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.232, }, }, "score": 0.232, "score_details": { "score": 0.232, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__curie-6-7b/helm_classic_openai_curie_6_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "mosaicml", "id": "mosaicml/mpt-30b", "name": "MPT 30B", }, "model_route_id": "mosaicml__mpt-30b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.231, }, }, "score": 0.231, "score_details": { "score": 0.231, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mosaicml__mpt-30b/helm_classic_mosaicml_mpt_30b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "aleph-alpha", "id": "aleph-alpha/luminous-supreme-70b", "name": "Luminous Supreme 70B", }, "model_route_id": "aleph-alpha__luminous-supreme-70b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.222, }, }, "score": 0.222, "score_details": { "score": 0.222, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/aleph-alpha__luminous-supreme-70b/helm_classic_aleph_alpha_luminous_supreme_70b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "aleph-alpha", "id": "aleph-alpha/luminous-extended-30b", "name": "Luminous Extended 30B", }, "model_route_id": "aleph-alpha__luminous-extended-30b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.221, }, }, "score": 0.221, "score_details": { "score": 0.221, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/aleph-alpha__luminous-extended-30b/helm_classic_aleph_alpha_luminous_extended_30b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "zhipu-ai", "id": "zhipu-ai/glm-130b", "name": "GLM 130B", }, "model_route_id": "zhipu-ai__glm-130b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.218, }, }, "score": 0.218, "score_details": { "score": 0.218, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/zhipu-ai__glm-130b/helm_classic_zhipu_ai_glm_130b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "cohere", "id": "cohere/cohere-small-v20220720-410m", "name": "Cohere small v20220720 410M", }, "model_route_id": "cohere__cohere-small-v20220720-410m", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.217, }, }, "score": 0.217, "score_details": { "score": 0.217, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__cohere-small-v20220720-410m/helm_classic_cohere_cohere_small_v20220720_410m_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/gpt-neox-20b", "name": "GPT-NeoX 20B", }, "model_route_id": "openai__gpt-neox-20b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.216, }, }, "score": 0.216, "score_details": { "score": 0.216, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-neox-20b/helm_classic_openai_gpt_neox_20b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/ada-350m", "name": "ada 350M", }, "model_route_id": "openai__ada-350m", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.215, }, }, "score": 0.215, "score_details": { "score": 0.215, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__ada-350m/helm_classic_openai_ada_350m_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "cohere", "id": "cohere/cohere-medium-v20221108-6-1b", "name": "Cohere medium v20221108 6.1B", }, "model_route_id": "cohere__cohere-medium-v20221108-6-1b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.215, }, }, "score": 0.215, "score_details": { "score": 0.215, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__cohere-medium-v20221108-6-1b/helm_classic_cohere_cohere_medium_v20221108_6_1b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-instruct-7b", "name": "Falcon-Instruct 7B", }, "model_route_id": "tiiuae__falcon-instruct-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.213, }, }, "score": 0.213, "score_details": { "score": 0.213, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/tiiuae__falcon-instruct-7b/helm_classic_tiiuae_falcon_instruct_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "eleutherai", "id": "eleutherai/pythia-6-9b", "name": "Pythia 6.9B", }, "model_route_id": "eleutherai__pythia-6-9b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.213, }, }, "score": 0.213, "score_details": { "score": 0.213, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/eleutherai__pythia-6-9b/helm_classic_eleutherai_pythia_6_9b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "together", "id": "together/redpajama-incite-instruct-v1-3b", "name": "RedPajama-INCITE-Instruct-v1 3B", }, "model_route_id": "together__redpajama-incite-instruct-v1-3b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.208, }, }, "score": 0.208, "score_details": { "score": 0.208, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/together__redpajama-incite-instruct-v1-3b/helm_classic_together_redpajama_incite_instruct_v1_3b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "together", "id": "together/redpajama-incite-base-7b", "name": "RedPajama-INCITE-Base 7B", }, "model_route_id": "together__redpajama-incite-base-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.205, }, }, "score": 0.205, "score_details": { "score": 0.205, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/together__redpajama-incite-base-7b/helm_classic_together_redpajama_incite_base_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "bigscience", "id": "bigscience/bloom-176b", "name": "BLOOM 176B", }, "model_route_id": "bigscience__bloom-176b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.205, }, }, "score": 0.205, "score_details": { "score": 0.205, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/bigscience__bloom-176b/helm_classic_bigscience_bloom_176b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "cohere", "id": "cohere/cohere-command-beta-6-1b", "name": "Cohere Command beta 6.1B", }, "model_route_id": "cohere__cohere-command-beta-6-1b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.203, }, }, "score": 0.203, "score_details": { "score": 0.203, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__cohere-command-beta-6-1b/helm_classic_cohere_cohere_command_beta_6_1b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "yandex", "id": "yandex/yalm-100b", "name": "YaLM 100B", }, "model_route_id": "yandex__yalm-100b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.202, }, }, "score": 0.202, "score_details": { "score": 0.202, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/yandex__yalm-100b/helm_classic_yandex_yalm_100b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "meta", "id": "meta/opt-66b", "name": "OPT 66B", }, "model_route_id": "meta__opt-66b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.201, }, }, "score": 0.201, "score_details": { "score": 0.201, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__opt-66b/helm_classic_meta_opt_66b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/gpt-j-6b", "name": "GPT-J 6B", }, "model_route_id": "openai__gpt-j-6b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.199, }, }, "score": 0.199, "score_details": { "score": 0.199, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-j-6b/helm_classic_openai_gpt_j_6b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "cohere", "id": "cohere/cohere-xlarge-v20220609-52-4b", "name": "Cohere xlarge v20220609 52.4B", }, "model_route_id": "cohere__cohere-xlarge-v20220609-52-4b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.198, }, }, "score": 0.198, "score_details": { "score": 0.198, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__cohere-xlarge-v20220609-52-4b/helm_classic_cohere_cohere_xlarge_v20220609_52_4b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "ai21", "id": "ai21/j1-large-v1-7-5b", "name": "J1-Large v1 7.5B", }, "model_route_id": "ai21__j1-large-v1-7-5b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.197, }, }, "score": 0.197, "score_details": { "score": 0.197, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__j1-large-v1-7-5b/helm_classic_ai21_j1_large_v1_7_5b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/davinci-175b", "name": "davinci 175B", }, "model_route_id": "openai__davinci-175b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.194, }, }, "score": 0.194, "score_details": { "score": 0.194, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__davinci-175b/helm_classic_openai_davinci_175b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "google", "id": "google/ul2-20b", "name": "UL2 20B", }, "model_route_id": "google__ul2-20b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.193, }, }, "score": 0.193, "score_details": { "score": 0.193, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__ul2-20b/helm_classic_google_ul2_20b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "ai21", "id": "ai21/j1-grande-v1-17b", "name": "J1-Grande v1 17B", }, "model_route_id": "ai21__j1-grande-v1-17b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.193, }, }, "score": 0.193, "score_details": { "score": 0.193, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__j1-grande-v1-17b/helm_classic_ai21_j1_grande_v1_17b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "cohere", "id": "cohere/cohere-medium-v20220720-6-1b", "name": "Cohere medium v20220720 6.1B", }, "model_route_id": "cohere__cohere-medium-v20220720-6-1b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.19, }, }, "score": 0.19, "score_details": { "score": 0.19, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__cohere-medium-v20220720-6-1b/helm_classic_cohere_cohere_medium_v20220720_6_1b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "openai", "id": "openai/babbage-1-3b", "name": "babbage 1.3B", }, "model_route_id": "openai__babbage-1-3b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.188, }, }, "score": 0.188, "score_details": { "score": 0.188, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__babbage-1-3b/helm_classic_openai_babbage_1_3b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "writer", "id": "writer/instructpalmyra-30b", "name": "InstructPalmyra 30B", }, "model_route_id": "writer__instructpalmyra-30b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.185, }, }, "score": 0.185, "score_details": { "score": 0.185, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__instructpalmyra-30b/helm_classic_writer_instructpalmyra_30b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "aleph-alpha", "id": "aleph-alpha/luminous-base-13b", "name": "Luminous Base 13B", }, "model_route_id": "aleph-alpha__luminous-base-13b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.182, }, }, "score": 0.182, "score_details": { "score": 0.182, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/aleph-alpha__luminous-base-13b/helm_classic_aleph_alpha_luminous_base_13b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "cohere", "id": "cohere/cohere-large-v20220720-13-1b", "name": "Cohere large v20220720 13.1B", }, "model_route_id": "cohere__cohere-large-v20220720-13-1b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.181, }, }, "score": 0.181, "score_details": { "score": 0.181, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__cohere-large-v20220720-13-1b/helm_classic_cohere_cohere_large_v20220720_13_1b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "eleutherai", "id": "eleutherai/pythia-12b", "name": "Pythia 12B", }, "model_route_id": "eleutherai__pythia-12b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.177, }, }, "score": 0.177, "score_details": { "score": 0.177, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/eleutherai__pythia-12b/helm_classic_eleutherai_pythia_12b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "ai21", "id": "ai21/j1-jumbo-v1-178b", "name": "J1-Jumbo v1 178B", }, "model_route_id": "ai21__j1-jumbo-v1-178b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.175, }, }, "score": 0.175, "score_details": { "score": 0.175, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__j1-jumbo-v1-178b/helm_classic_ai21_j1_jumbo_v1_178b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "cohere", "id": "cohere/cohere-xlarge-v20221108-52-4b", "name": "Cohere xlarge v20221108 52.4B", }, "model_route_id": "cohere__cohere-xlarge-v20221108-52-4b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.169, }, }, "score": 0.169, "score_details": { "score": 0.169, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__cohere-xlarge-v20221108-52-4b/helm_classic_cohere_cohere_xlarge_v20221108_52_4b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "microsoft", "id": "microsoft/tnlg-v2-6-7b", "name": "TNLG v2 6.7B", }, "model_route_id": "microsoft__tnlg-v2-6-7b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.167, }, }, "score": 0.167, "score_details": { "score": 0.167, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/microsoft__tnlg-v2-6-7b/helm_classic_microsoft_tnlg_v2_6_7b_1774096308_339228.json", }, { "evaluation_timestamp": "1774096308.339228", "model_info": { "developer": "google", "id": "google/t5-11b", "name": "T5 11B", }, "model_route_id": "google__t5-11b", "result": { "canonical_display_name": "TruthfulQA / Exact Match", "detailed_evaluation_results_url": undefined, "display_name": "TruthfulQA / Exact Match", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "Exact Match", "evaluation_timestamp": "1774096308.339228", "metric_config": { "evaluation_description": "EM on TruthfulQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "exact_match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "score_details": { "score": 0.133, }, }, "score": 0.133, "score_details": { "score": 0.133, }, "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_classic", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__t5-11b/helm_classic_google_t5_11b_1774096308_339228.json", }, ], "models_count": 67, "provenance_summary": { "first_party_only_groups": 0, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 0, "third_party": 67, "unspecified": 0, }, "total_groups": 67, "total_results": 67, }, "reproducibility_summary": { "has_reproducibility_gap_count": 67, "populated_ratio_avg": 0, "results_total": 67, }, "root_metrics": [ { "canonical_display_name": "TruthfulQA / Exact Match", "display_name": "TruthfulQA / Exact Match", "lower_is_better": false, "metric_key": "exact_match", "metric_name": "Exact Match", "metric_summary_id": "helm_classic_truthfulqa_exact_match", "models_count": 67, "top_score": 0.616, "unit": "proportion", }, ], "source_data": { "dataset_name": "TruthfulQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json", ], }, "source_types": [], "subtasks": [], "third_party_ratio": 0, "worst_model": { "name": "T5 11B", "score": 0.133, }, } `; exports[`hfEvalDetailToSummary > helm_lite_narrativeqa — Subtask leaf under helm_lite parent; pipeline category=reasoning, regex returns General 1`] = ` { "avg_score": 0.6938461538461538, "avg_score_norm": 0.6938461538461538, "benchmark_card": { "benchmark_details": { "benchmark_type": "composite", "contains": [ "GSM8K", "LegalBench", "MATH", "MMLU", "MedQA", "NarrativeQA", "NaturalQuestions (closed-book)", "OpenbookQA", "WMT 2014", ], "data_type": "composite", "domains": [ "mathematical reasoning", "legal analysis", "multilingual understanding", "medical knowledge", "narrative comprehension", "question answering", ], "languages": [ "English", "multiple", ], "name": "helm_lite", "overview": "helm_lite is a composite benchmark suite designed to provide a comprehensive evaluation of AI models across multiple domains and capabilities. It aggregates performance metrics from nine distinct sub-benchmarks: GSM8K, LegalBench, MATH, MMLU, MedQA, NarrativeQA, NaturalQuestions (closed-book), OpenbookQA, and WMT 2014. The suite measures overall model proficiency through aggregated scoring across these diverse tasks, covering areas such as mathematical reasoning, legal analysis, multilingual understanding, medical knowledge, narrative comprehension, and question answering.", "resources": [ "https://crfm.stanford.edu/helm/lite/latest/", "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", "https://arxiv.org/abs/2211.09110", ], "similar_benchmarks": [ "Not specified", ], }, "data": { "annotation": "Not specified", "format": "Not specified", "size": "Not specified", "source": "The HELM-Lite benchmark is a composite suite that aggregates multiple specialized benchmarks to measure overall AI performance across diverse domains including reasoning, knowledge, and language understanding. It integrates GSM8K (math), LegalBench (legal reasoning), MATH (advanced math), MMLU (multidisciplinary knowledge), MedQA (medical knowledge), NarrativeQA (story understanding), NaturalQuestions (closed-book factoid QA), OpenbookQA (commonsense reasoning), and WMT 2014 (machine translation).", }, "ethical_and_legal_considerations": { "compliance_with_regulations": "Not specified", "consent_procedures": "Not specified", "data_licensing": "Not specified", "privacy_and_anonymity": "Not specified", }, "flagged_fields": {}, "methodology": { "baseline_results": "Not specified", "calculation": "Scores are calculated independently for each sub-benchmark using their respective metrics and aggregated at the suite level. All metrics are continuous with higher scores indicating better performance.", "interpretation": "Higher scores across all metrics indicate better overall performance. The suite provides comprehensive coverage across mathematical, legal, medical, commonsense reasoning, reading comprehension, and translation capabilities.", "methods": [ "The suite evaluates AI models across multiple reasoning domains using a composite of established benchmarks", "Each sub-benchmark measures specific capabilities including mathematical reasoning, legal analysis, medical knowledge, reading comprehension, and machine translation", ], "metrics": [ "Exact Match (EM) for GSM8K, LegalBench, MMLU, MedQA, and OpenbookQA", "F1 score for NarrativeQA and NaturalQuestions (closed-book)", "Equivalent (CoT) for MATH", "BLEU-4 for WMT 2014", ], "validation": "Not specified", }, "missing_fields": [], "possible_risks": [ { "category": "Over- or under-reliance", "description": [ "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html", }, { "category": "Unrepresentative data", "description": [ "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html", }, { "category": "Uncertain data provenance", "description": [ "Data provenance refers to the traceability of data (including synthetic data), which includes its ownership, origin, transformations, and generation. Proving that the data is the same as the original source with correct usage terms is difficult without standardized methods for verifying data sources or generation.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-provenance.html", }, { "category": "Data contamination", "description": [ "Data contamination occurs when incorrect data is used for training. For example, data that is not aligned with model's purpose or data that is already set aside for other development tasks such as testing and evaluation.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-contamination.html", }, { "category": "Reproducibility", "description": [ "Replicating agent behavior or output can be impacted by changes or updates made to external services and tools. This impact is increased if the agent is built with generative AI.", ], "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/reproducibility-agentic.html", }, ], "purpose_and_intended_users": { "audience": [ "AI researchers", "Model developers", ], "goal": "To provide a comprehensive evaluation of AI models across multiple reasoning domains and knowledge areas through a standardized composite benchmark suite.", "limitations": "Not specified", "out_of_scope_uses": [ "Not specified", ], "tasks": [ "Mathematical reasoning", "Legal reasoning", "Medical question answering", "General knowledge evaluation", "Reading comprehension", "Machine translation", ], }, }, "best_model": { "name": "GPT-4o 2024-05-13", "score": 0.804, }, "canonical_display_name": "NarrativeQA", "category": "General", "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 0, "total_groups": 89, "variant_divergent_count": 0, }, "composite_benchmark_key": "NarrativeQA", "composite_benchmark_name": "NarrativeQA", "evalcards": { "annotations": { "benchmark_comparability": { "cross_party_divergence_groups": [], "variant_divergence_groups": [], }, "reporting_completeness": { "completeness_score": 0.9285714285714286, "field_scores": [ { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.name", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.overview", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.data_type", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.domains", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.languages", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.similar_benchmarks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.resources", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.goal", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.audience", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.tasks", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.limitations", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.methods", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.metrics", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.calculation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.interpretation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.baseline_results", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.validation", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "score": 1, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "score": 1, }, { "coverage_type": "partial", "field_path": "autobenchmarkcard.data", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_type", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_organization_name", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.evaluator_relationship", "score": 1, }, { "coverage_type": "reserved", "field_path": "evalcards.lifecycle_status", "score": 0, }, { "coverage_type": "reserved", "field_path": "evalcards.preregistration_url", "score": 0, }, ], "missing_required_fields": [ "evalcards.lifecycle_status", "evalcards.preregistration_url", ], "partial_fields": [], "signal_version": "1.0", "total_fields_evaluated": 28, }, }, }, "evaluation_id": "helm_lite_narrativeqa", "evaluation_name": "NarrativeQA", "evaluator_names": [], "latest_source_name": "NarrativeQA", "leaderboard_metrics": [ { "canonical_display_name": "NarrativeQA / F1", "column_key": "root:helm_lite_narrativeqa_f1", "display_name": "NarrativeQA / F1", "lower_is_better": false, "metric_name": "F1", "metric_summary_id": "helm_lite_narrativeqa_f1", "scope": "root", "subtask_key": undefined, "subtask_name": undefined, "unit": "proportion", }, ], "leaderboard_rows": [ { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o 2024-05-13", }, "model_route_id": "openai__gpt-4o", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.795, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-70b", "name": "Llama 3 70B", }, "model_route_id": "meta__llama-3-70b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.798, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-v3", "name": "DeepSeek v3", }, "model_route_id": "deepseek-ai__deepseek-v3", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.796, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-3-70b-instruct-turbo", "name": "Llama 3.3 Instruct Turbo 70B", }, "model_route_id": "meta__llama-3-3-70b-instruct-turbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.791, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "amazon", "id": "amazon/nova-pro-v1-0", "name": "Amazon Nova Pro", }, "model_route_id": "amazon__nova-pro-v1-0", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.791, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-2-27b-it", "name": "Gemma 2 Instruct 27B", }, "model_route_id": "google__gemma-2-27b-it", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.79, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-exp", "name": "Gemini 2.0 Flash Experimental", }, "model_route_id": "google__gemini-2-0-flash-exp", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.783, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-001", "name": "Gemini 1.5 Pro 001", }, "model_route_id": "google__gemini-1-5-pro-001", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.783, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-001", "name": "Gemini 1.5 Flash 001", }, "model_route_id": "google__gemini-1-5-flash-001", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.783, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "01-ai", "id": "01-ai/yi-34b", "name": "Yi 34B", }, "model_route_id": "01-ai__yi-34b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.782, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x22b", "name": "Mixtral 8x22B", }, "model_route_id": "mistralai__mixtral-8x22b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.779, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-large-2407", "name": "Mistral Large 2 2407", }, "model_route_id": "mistralai__mistral-large-2407", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.779, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-2-90b-vision-instruct-turbo", "name": "Llama 3.2 Vision Instruct Turbo 90B", }, "model_route_id": "meta__llama-3-2-90b-vision-instruct-turbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.777, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-x-004", "name": "Palmyra-X-004", }, "model_route_id": "writer__palmyra-x-004", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.773, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-70b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 70B", }, "model_route_id": "meta__llama-3-1-70b-instruct-turbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.772, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-sonnet", "name": "Claude 3.5 Sonnet 20241022", }, "model_route_id": "anthropic__claude-3-5-sonnet", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.746, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o-mini", "name": "GPT-4o mini 2024-07-18", }, "model_route_id": "openai__gpt-4o-mini", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.768, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-0613", "name": "GPT-4 0613", }, "model_route_id": "openai__gpt-4-0613", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.768, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-2-9b-it", "name": "Gemma 2 Instruct 9B", }, "model_route_id": "google__gemma-2-9b-it", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.768, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "amazon", "id": "amazon/nova-lite-v1-0", "name": "Amazon Nova Lite", }, "model_route_id": "amazon__nova-lite-v1-0", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.768, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x7b-32kseqlen", "name": "Mixtral 8x7B 32K seqlen", }, "model_route_id": "mistralai__mixtral-8x7b-32kseqlen", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.767, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-2-70b", "name": "Llama 2 70B", }, "model_route_id": "meta__llama-2-70b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.763, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-haiku", "name": "Claude 3.5 Haiku 20241022", }, "model_route_id": "anthropic__claude-3-5-haiku", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.763, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-turbo", "name": "GPT-4 Turbo 2024-04-09", }, "model_route_id": "openai__gpt-4-turbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.761, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-2-11b-vision-instruct-turbo", "name": "Llama 3.2 Vision Instruct Turbo 11B", }, "model_route_id": "meta__llama-3-2-11b-vision-instruct-turbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.756, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-8b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 8B", }, "model_route_id": "meta__llama-3-1-8b-instruct-turbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.756, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-002", "name": "Gemini 1.5 Pro 002", }, "model_route_id": "google__gemini-1-5-pro-002", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.756, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-65b", "name": "LLaMA 65B", }, "model_route_id": "meta__llama-65b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.755, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "microsoft", "id": "microsoft/phi-3-small-8k-instruct", "name": "Phi-3 7B", }, "model_route_id": "microsoft__phi-3-small-8k-instruct", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.754, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-8b", "name": "Llama 3 8B", }, "model_route_id": "meta__llama-3-8b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.754, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "upstage", "id": "upstage/solar-pro-241126", "name": "Solar Pro", }, "model_route_id": "upstage__solar-pro-241126", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.753, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-x-v2", "name": "Palmyra X V2 33B", }, "model_route_id": "writer__palmyra-x-v2", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.752, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemma-7b", "name": "Gemma 7B", }, "model_route_id": "google__gemma-7b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.752, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-0-pro-002", "name": "Gemini 1.0 Pro 002", }, "model_route_id": "google__gemini-1-0-pro-002", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.751, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-405b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 405B", }, "model_route_id": "meta__llama-3-1-405b-instruct-turbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.749, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/command", "name": "Command", }, "model_route_id": "cohere__command", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.749, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-002", "name": "Gemini 1.5 Flash 002", }, "model_route_id": "google__gemini-1-5-flash-002", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.746, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/jamba-1-5-mini", "name": "Jamba 1.5 Mini", }, "model_route_id": "ai21__jamba-1-5-mini", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.746, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-72b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 72B", }, "model_route_id": "qwen__qwen2-5-72b-instruct-turbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.745, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "amazon", "id": "amazon/nova-micro-v1-0", "name": "Amazon Nova Micro", }, "model_route_id": "amazon__nova-micro-v1-0", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.744, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/j2-grande", "name": "Jurassic-2 Grande 17B", }, "model_route_id": "ai21__j2-grande", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.744, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "AlephAlpha", "id": "alephalpha/luminous-supreme", "name": "Luminous Supreme 70B", }, "model_route_id": "alephalpha__luminous-supreme", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.743, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-7b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 7B", }, "model_route_id": "qwen__qwen2-5-7b-instruct-turbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.742, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/command-r", "name": "Command R", }, "model_route_id": "cohere__command-r", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.742, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-2-13b", "name": "Llama 2 13B", }, "model_route_id": "meta__llama-2-13b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.741, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/command-r-plus", "name": "Command R Plus", }, "model_route_id": "cohere__command-r-plus", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.735, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/text-davinci-003", "name": "GPT-3.5 text-davinci-003", }, "model_route_id": "openai__text-davinci-003", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.731, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/open-mistral-nemo-2407", "name": "Mistral NeMo 2402", }, "model_route_id": "mistralai__open-mistral-nemo-2407", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.731, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/j2-jumbo", "name": "Jurassic-2 Jumbo 178B", }, "model_route_id": "ai21__j2-jumbo", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.728, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen2-72b-instruct", "name": "Qwen2 Instruct 72B", }, "model_route_id": "qwen__qwen2-72b-instruct", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.727, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1106-preview", "name": "GPT-4 Turbo 1106 preview", }, "model_route_id": "openai__gpt-4-1106-preview", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.727, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "microsoft", "id": "microsoft/phi-3-medium-4k-instruct", "name": "Phi-3 14B", }, "model_route_id": "microsoft__phi-3-medium-4k-instruct", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.724, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-v1-3", "name": "Claude v1.3", }, "model_route_id": "anthropic__claude-v1-3", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.723, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-110b-chat", "name": "Qwen1.5 Chat 110B", }, "model_route_id": "qwen__qwen1-5-110b-chat", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.721, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/text-davinci-002", "name": "GPT-3.5 text-davinci-002", }, "model_route_id": "openai__text-davinci-002", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.719, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/text-bison-001", "name": "PaLM-2 Bison", }, "model_route_id": "google__text-bison-001", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.718, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-2-0", "name": "Claude 2.0", }, "model_route_id": "anthropic__claude-2-0", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.718, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-v0-1", "name": "Mistral v0.1 7B", }, "model_route_id": "mistralai__mistral-7b-v0-1", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.716, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-instruct-v0-3", "name": "Mistral Instruct v0.3 7B", }, "model_route_id": "mistralai__mistral-7b-instruct-v0-3", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.716, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-14b", "name": "Qwen1.5 14B", }, "model_route_id": "qwen__qwen1-5-14b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.711, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-x-v3", "name": "Palmyra X V3 72B", }, "model_route_id": "writer__palmyra-x-v3", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.706, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "microsoft", "id": "microsoft/phi-2", "name": "Phi-2", }, "model_route_id": "microsoft__phi-2", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.703, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "01-ai", "id": "01-ai/yi-6b", "name": "Yi 6B", }, "model_route_id": "01-ai__yi-6b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.702, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-2-7b", "name": "Llama 2 7B", }, "model_route_id": "meta__llama-2-7b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.686, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "AlephAlpha", "id": "alephalpha/luminous-extended", "name": "Luminous Extended 30B", }, "model_route_id": "alephalpha__luminous-extended", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.684, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-2-1", "name": "Claude 2.1", }, "model_route_id": "anthropic__claude-2-1", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.677, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-40b", "name": "Falcon 40B", }, "model_route_id": "tiiuae__falcon-40b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.671, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/jamba-1-5-large", "name": "Jamba 1.5 Large", }, "model_route_id": "ai21__jamba-1-5-large", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.664, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "ai21", "id": "ai21/jamba-instruct", "name": "Jamba Instruct", }, "model_route_id": "ai21__jamba-instruct", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.658, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0613", "name": "GPT-3.5 Turbo 0613", }, "model_route_id": "openai__gpt-3-5-turbo-0613", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.655, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "snowflake", "id": "snowflake/snowflake-arctic-instruct", "name": "Arctic Instruct", }, "model_route_id": "snowflake__snowflake-arctic-instruct", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.654, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "AlephAlpha", "id": "alephalpha/luminous-base", "name": "Luminous Base 13B", }, "model_route_id": "alephalpha__luminous-base", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.633, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/command-light", "name": "Command Light", }, "model_route_id": "cohere__command-light", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.629, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-7b", "name": "Falcon 7B", }, "model_route_id": "tiiuae__falcon-7b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.621, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-instant-1-2", "name": "Claude Instant 1.2", }, "model_route_id": "anthropic__claude-instant-1-2", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.616, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-72b", "name": "Qwen1.5 72B", }, "model_route_id": "qwen__qwen1-5-72b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.601, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "allenai", "id": "allenai/olmo-7b", "name": "OLMo 7B", }, "model_route_id": "allenai__olmo-7b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.597, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-32b", "name": "Qwen1.5 32B", }, "model_route_id": "qwen__qwen1-5-32b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.589, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/text-unicorn-001", "name": "PaLM-2 Unicorn", }, "model_route_id": "google__text-unicorn-001", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.583, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-llm-67b-chat", "name": "DeepSeek LLM Chat 67B", }, "model_route_id": "deepseek-ai__deepseek-llm-67b-chat", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.581, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-small-2402", "name": "Mistral Small 2402", }, "model_route_id": "mistralai__mistral-small-2402", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.519, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "databricks", "id": "databricks/dbrx-instruct", "name": "DBRX Instruct", }, "model_route_id": "databricks__dbrx-instruct", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.488, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-large-2402", "name": "Mistral Large 2402", }, "model_route_id": "mistralai__mistral-large-2402", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.454, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-medium-2312", "name": "Mistral Medium 2312", }, "model_route_id": "mistralai__mistral-medium-2312", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.449, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-7b", "name": "Qwen1.5 7B", }, "model_route_id": "qwen__qwen1-5-7b", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.448, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "01-ai", "id": "01-ai/yi-large-preview", "name": "Yi Large Preview", }, "model_route_id": "01-ai__yi-large-preview", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.373, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-opus", "name": "Claude 3 Opus 20240229", }, "model_route_id": "anthropic__claude-3-opus", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.351, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-haiku", "name": "Claude 3 Haiku 20240307", }, "model_route_id": "anthropic__claude-3-haiku", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.244, }, }, { "annotations_by_metric": { "root:helm_lite_narrativeqa_f1": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1774096306.427425", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-sonnet", "name": "Claude 3 Sonnet 20240229", }, "model_route_id": "anthropic__claude-3-sonnet", "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_lite_narrativeqa_f1": 0.111, }, }, ], "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_names": [ "F1", ], "metrics_count": 1, "missing_generation_config_count": 0, "model_results": [ { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o 2024-05-13", }, "model_route_id": "openai__gpt-4o", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.804, }, }, "score": 0.804, "score_details": { "score": 0.804, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o/helm_lite_openai_gpt_4o_2024_05_13_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-3-70b", "name": "Llama 3 70B", }, "model_route_id": "meta__llama-3-70b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.798, }, }, "score": 0.798, "score_details": { "score": 0.798, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-70b/helm_lite_meta_llama_3_70b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-v3", "name": "DeepSeek v3", }, "model_route_id": "deepseek-ai__deepseek-v3", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.796, }, }, "score": 0.796, "score_details": { "score": 0.796, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek-ai__deepseek-v3/helm_lite_deepseek_ai_deepseek_v3_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o 2024-08-06", }, "model_route_id": "openai__gpt-4o", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.795, }, }, "score": 0.795, "score_details": { "score": 0.795, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o/helm_lite_openai_gpt_4o_2024_08_06_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-3-3-70b-instruct-turbo", "name": "Llama 3.3 Instruct Turbo 70B", }, "model_route_id": "meta__llama-3-3-70b-instruct-turbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.791, }, }, "score": 0.791, "score_details": { "score": 0.791, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-3-70b-instruct-turbo/helm_lite_meta_llama_3_3_70b_instruct_turbo_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "amazon", "id": "amazon/nova-pro-v1-0", "name": "Amazon Nova Pro", }, "model_route_id": "amazon__nova-pro-v1-0", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.791, }, }, "score": 0.791, "score_details": { "score": 0.791, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/amazon__nova-pro-v1-0/helm_lite_amazon_nova_pro_v1_0_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/gemma-2-27b-it", "name": "Gemma 2 Instruct 27B", }, "model_route_id": "google__gemma-2-27b-it", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.79, }, }, "score": 0.79, "score_details": { "score": 0.79, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-2-27b-it/helm_lite_google_gemma_2_27b_it_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-exp", "name": "Gemini 2.0 Flash Experimental", }, "model_route_id": "google__gemini-2-0-flash-exp", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.783, }, }, "score": 0.783, "score_details": { "score": 0.783, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash-exp/helm_lite_google_gemini_2_0_flash_exp_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-001", "name": "Gemini 1.5 Pro 001", }, "model_route_id": "google__gemini-1-5-pro-001", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.783, }, }, "score": 0.783, "score_details": { "score": 0.783, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-pro-001/helm_lite_google_gemini_1_5_pro_001_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-001", "name": "Gemini 1.5 Flash 001", }, "model_route_id": "google__gemini-1-5-flash-001", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.783, }, }, "score": 0.783, "score_details": { "score": 0.783, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-flash-001/helm_lite_google_gemini_1_5_flash_001_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "01-ai", "id": "01-ai/yi-34b", "name": "Yi 34B", }, "model_route_id": "01-ai__yi-34b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.782, }, }, "score": 0.782, "score_details": { "score": 0.782, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/01-ai__yi-34b/helm_lite_01_ai_yi_34b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x22b", "name": "Mixtral 8x22B", }, "model_route_id": "mistralai__mixtral-8x22b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.779, }, }, "score": 0.779, "score_details": { "score": 0.779, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mixtral-8x22b/helm_lite_mistralai_mixtral_8x22b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-large-2407", "name": "Mistral Large 2 2407", }, "model_route_id": "mistralai__mistral-large-2407", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.779, }, }, "score": 0.779, "score_details": { "score": 0.779, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-large-2407/helm_lite_mistralai_mistral_large_2407_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-3-2-90b-vision-instruct-turbo", "name": "Llama 3.2 Vision Instruct Turbo 90B", }, "model_route_id": "meta__llama-3-2-90b-vision-instruct-turbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.777, }, }, "score": 0.777, "score_details": { "score": 0.777, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-2-90b-vision-instruct-turbo/helm_lite_meta_llama_3_2_90b_vision_instruct_turbo_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "writer", "id": "writer/palmyra-x-004", "name": "Palmyra-X-004", }, "model_route_id": "writer__palmyra-x-004", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.773, }, }, "score": 0.773, "score_details": { "score": 0.773, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-x-004/helm_lite_writer_palmyra_x_004_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-3-1-70b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 70B", }, "model_route_id": "meta__llama-3-1-70b-instruct-turbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.772, }, }, "score": 0.772, "score_details": { "score": 0.772, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-70b-instruct-turbo/helm_lite_meta_llama_3_1_70b_instruct_turbo_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-sonnet", "name": "Claude 3.5 Sonnet 20241022", }, "model_route_id": "anthropic__claude-3-5-sonnet", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.77, }, }, "score": 0.77, "score_details": { "score": 0.77, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-5-sonnet/helm_lite_anthropic_claude_3_5_sonnet_20241022_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "openai", "id": "openai/gpt-4o-mini", "name": "GPT-4o mini 2024-07-18", }, "model_route_id": "openai__gpt-4o-mini", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.768, }, }, "score": 0.768, "score_details": { "score": 0.768, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o-mini/helm_lite_openai_gpt_4o_mini_2024_07_18_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "openai", "id": "openai/gpt-4-0613", "name": "GPT-4 0613", }, "model_route_id": "openai__gpt-4-0613", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.768, }, }, "score": 0.768, "score_details": { "score": 0.768, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-0613/helm_lite_openai_gpt_4_0613_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/gemma-2-9b-it", "name": "Gemma 2 Instruct 9B", }, "model_route_id": "google__gemma-2-9b-it", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.768, }, }, "score": 0.768, "score_details": { "score": 0.768, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-2-9b-it/helm_lite_google_gemma_2_9b_it_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "amazon", "id": "amazon/nova-lite-v1-0", "name": "Amazon Nova Lite", }, "model_route_id": "amazon__nova-lite-v1-0", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.768, }, }, "score": 0.768, "score_details": { "score": 0.768, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/amazon__nova-lite-v1-0/helm_lite_amazon_nova_lite_v1_0_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x7b-32kseqlen", "name": "Mixtral 8x7B 32K seqlen", }, "model_route_id": "mistralai__mixtral-8x7b-32kseqlen", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.767, }, }, "score": 0.767, "score_details": { "score": 0.767, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mixtral-8x7b-32kseqlen/helm_lite_mistralai_mixtral_8x7b_32kseqlen_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-2-70b", "name": "Llama 2 70B", }, "model_route_id": "meta__llama-2-70b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.763, }, }, "score": 0.763, "score_details": { "score": 0.763, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-2-70b/helm_lite_meta_llama_2_70b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-haiku", "name": "Claude 3.5 Haiku 20241022", }, "model_route_id": "anthropic__claude-3-5-haiku", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.763, }, }, "score": 0.763, "score_details": { "score": 0.763, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-5-haiku/helm_lite_anthropic_claude_3_5_haiku_20241022_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "openai", "id": "openai/gpt-4-turbo", "name": "GPT-4 Turbo 2024-04-09", }, "model_route_id": "openai__gpt-4-turbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.761, }, }, "score": 0.761, "score_details": { "score": 0.761, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-turbo/helm_lite_openai_gpt_4_turbo_2024_04_09_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-3-2-11b-vision-instruct-turbo", "name": "Llama 3.2 Vision Instruct Turbo 11B", }, "model_route_id": "meta__llama-3-2-11b-vision-instruct-turbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.756, }, }, "score": 0.756, "score_details": { "score": 0.756, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-2-11b-vision-instruct-turbo/helm_lite_meta_llama_3_2_11b_vision_instruct_turbo_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-3-1-8b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 8B", }, "model_route_id": "meta__llama-3-1-8b-instruct-turbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.756, }, }, "score": 0.756, "score_details": { "score": 0.756, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-8b-instruct-turbo/helm_lite_meta_llama_3_1_8b_instruct_turbo_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-002", "name": "Gemini 1.5 Pro 002", }, "model_route_id": "google__gemini-1-5-pro-002", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.756, }, }, "score": 0.756, "score_details": { "score": 0.756, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-pro-002/helm_lite_google_gemini_1_5_pro_002_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-65b", "name": "LLaMA 65B", }, "model_route_id": "meta__llama-65b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.755, }, }, "score": 0.755, "score_details": { "score": 0.755, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-65b/helm_lite_meta_llama_65b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "microsoft", "id": "microsoft/phi-3-small-8k-instruct", "name": "Phi-3 7B", }, "model_route_id": "microsoft__phi-3-small-8k-instruct", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.754, }, }, "score": 0.754, "score_details": { "score": 0.754, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/microsoft__phi-3-small-8k-instruct/helm_lite_microsoft_phi_3_small_8k_instruct_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-3-8b", "name": "Llama 3 8B", }, "model_route_id": "meta__llama-3-8b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.754, }, }, "score": 0.754, "score_details": { "score": 0.754, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-8b/helm_lite_meta_llama_3_8b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "upstage", "id": "upstage/solar-pro-241126", "name": "Solar Pro", }, "model_route_id": "upstage__solar-pro-241126", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.753, }, }, "score": 0.753, "score_details": { "score": 0.753, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/upstage__solar-pro-241126/helm_lite_upstage_solar_pro_241126_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "writer", "id": "writer/palmyra-x-v2", "name": "Palmyra X V2 33B", }, "model_route_id": "writer__palmyra-x-v2", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.752, }, }, "score": 0.752, "score_details": { "score": 0.752, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-x-v2/helm_lite_writer_palmyra_x_v2_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/gemma-7b", "name": "Gemma 7B", }, "model_route_id": "google__gemma-7b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.752, }, }, "score": 0.752, "score_details": { "score": 0.752, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemma-7b/helm_lite_google_gemma_7b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/gemini-1-0-pro-002", "name": "Gemini 1.0 Pro 002", }, "model_route_id": "google__gemini-1-0-pro-002", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.751, }, }, "score": 0.751, "score_details": { "score": 0.751, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-0-pro-002/helm_lite_google_gemini_1_0_pro_002_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-3-1-405b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 405B", }, "model_route_id": "meta__llama-3-1-405b-instruct-turbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.749, }, }, "score": 0.749, "score_details": { "score": 0.749, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-405b-instruct-turbo/helm_lite_meta_llama_3_1_405b_instruct_turbo_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "cohere", "id": "cohere/command", "name": "Command", }, "model_route_id": "cohere__command", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.749, }, }, "score": 0.749, "score_details": { "score": 0.749, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__command/helm_lite_cohere_command_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-002", "name": "Gemini 1.5 Flash 002", }, "model_route_id": "google__gemini-1-5-flash-002", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.746, }, }, "score": 0.746, "score_details": { "score": 0.746, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-flash-002/helm_lite_google_gemini_1_5_flash_002_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-sonnet", "name": "Claude 3.5 Sonnet 20240620", }, "model_route_id": "anthropic__claude-3-5-sonnet", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.746, }, }, "score": 0.746, "score_details": { "score": 0.746, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-5-sonnet/helm_lite_anthropic_claude_3_5_sonnet_20240620_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "ai21", "id": "ai21/jamba-1-5-mini", "name": "Jamba 1.5 Mini", }, "model_route_id": "ai21__jamba-1-5-mini", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.746, }, }, "score": 0.746, "score_details": { "score": 0.746, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__jamba-1-5-mini/helm_lite_ai21_jamba_1_5_mini_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-72b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 72B", }, "model_route_id": "qwen__qwen2-5-72b-instruct-turbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.745, }, }, "score": 0.745, "score_details": { "score": 0.745, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen2-5-72b-instruct-turbo/helm_lite_qwen_qwen2_5_72b_instruct_turbo_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "amazon", "id": "amazon/nova-micro-v1-0", "name": "Amazon Nova Micro", }, "model_route_id": "amazon__nova-micro-v1-0", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.744, }, }, "score": 0.744, "score_details": { "score": 0.744, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/amazon__nova-micro-v1-0/helm_lite_amazon_nova_micro_v1_0_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "ai21", "id": "ai21/j2-grande", "name": "Jurassic-2 Grande 17B", }, "model_route_id": "ai21__j2-grande", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.744, }, }, "score": 0.744, "score_details": { "score": 0.744, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__j2-grande/helm_lite_ai21_j2_grande_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "AlephAlpha", "id": "alephalpha/luminous-supreme", "name": "Luminous Supreme 70B", }, "model_route_id": "alephalpha__luminous-supreme", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.743, }, }, "score": 0.743, "score_details": { "score": 0.743, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alephalpha__luminous-supreme/helm_lite_alephalpha_luminous_supreme_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-7b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 7B", }, "model_route_id": "qwen__qwen2-5-7b-instruct-turbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.742, }, }, "score": 0.742, "score_details": { "score": 0.742, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen2-5-7b-instruct-turbo/helm_lite_qwen_qwen2_5_7b_instruct_turbo_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "cohere", "id": "cohere/command-r", "name": "Command R", }, "model_route_id": "cohere__command-r", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.742, }, }, "score": 0.742, "score_details": { "score": 0.742, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__command-r/helm_lite_cohere_command_r_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-2-13b", "name": "Llama 2 13B", }, "model_route_id": "meta__llama-2-13b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.741, }, }, "score": 0.741, "score_details": { "score": 0.741, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-2-13b/helm_lite_meta_llama_2_13b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "cohere", "id": "cohere/command-r-plus", "name": "Command R Plus", }, "model_route_id": "cohere__command-r-plus", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.735, }, }, "score": 0.735, "score_details": { "score": 0.735, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__command-r-plus/helm_lite_cohere_command_r_plus_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "openai", "id": "openai/text-davinci-003", "name": "GPT-3.5 text-davinci-003", }, "model_route_id": "openai__text-davinci-003", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.731, }, }, "score": 0.731, "score_details": { "score": 0.731, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__text-davinci-003/helm_lite_openai_text_davinci_003_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "mistralai", "id": "mistralai/open-mistral-nemo-2407", "name": "Mistral NeMo 2402", }, "model_route_id": "mistralai__open-mistral-nemo-2407", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.731, }, }, "score": 0.731, "score_details": { "score": 0.731, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__open-mistral-nemo-2407/helm_lite_mistralai_open_mistral_nemo_2407_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "ai21", "id": "ai21/j2-jumbo", "name": "Jurassic-2 Jumbo 178B", }, "model_route_id": "ai21__j2-jumbo", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.728, }, }, "score": 0.728, "score_details": { "score": 0.728, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__j2-jumbo/helm_lite_ai21_j2_jumbo_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "qwen", "id": "qwen/qwen2-72b-instruct", "name": "Qwen2 Instruct 72B", }, "model_route_id": "qwen__qwen2-72b-instruct", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.727, }, }, "score": 0.727, "score_details": { "score": 0.727, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen2-72b-instruct/helm_lite_qwen_qwen2_72b_instruct_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "openai", "id": "openai/gpt-4-1106-preview", "name": "GPT-4 Turbo 1106 preview", }, "model_route_id": "openai__gpt-4-1106-preview", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.727, }, }, "score": 0.727, "score_details": { "score": 0.727, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1106-preview/helm_lite_openai_gpt_4_1106_preview_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "microsoft", "id": "microsoft/phi-3-medium-4k-instruct", "name": "Phi-3 14B", }, "model_route_id": "microsoft__phi-3-medium-4k-instruct", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.724, }, }, "score": 0.724, "score_details": { "score": 0.724, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/microsoft__phi-3-medium-4k-instruct/helm_lite_microsoft_phi_3_medium_4k_instruct_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-v1-3", "name": "Claude v1.3", }, "model_route_id": "anthropic__claude-v1-3", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.723, }, }, "score": 0.723, "score_details": { "score": 0.723, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-v1-3/helm_lite_anthropic_claude_v1_3_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-110b-chat", "name": "Qwen1.5 Chat 110B", }, "model_route_id": "qwen__qwen1-5-110b-chat", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.721, }, }, "score": 0.721, "score_details": { "score": 0.721, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen1-5-110b-chat/helm_lite_qwen_qwen1_5_110b_chat_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "openai", "id": "openai/text-davinci-002", "name": "GPT-3.5 text-davinci-002", }, "model_route_id": "openai__text-davinci-002", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.719, }, }, "score": 0.719, "score_details": { "score": 0.719, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__text-davinci-002/helm_lite_openai_text_davinci_002_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/text-bison-001", "name": "PaLM-2 Bison", }, "model_route_id": "google__text-bison-001", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.718, }, }, "score": 0.718, "score_details": { "score": 0.718, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__text-bison-001/helm_lite_google_text_bison_001_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-2-0", "name": "Claude 2.0", }, "model_route_id": "anthropic__claude-2-0", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.718, }, }, "score": 0.718, "score_details": { "score": 0.718, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-2-0/helm_lite_anthropic_claude_2_0_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-v0-1", "name": "Mistral v0.1 7B", }, "model_route_id": "mistralai__mistral-7b-v0-1", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.716, }, }, "score": 0.716, "score_details": { "score": 0.716, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-7b-v0-1/helm_lite_mistralai_mistral_7b_v0_1_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-instruct-v0-3", "name": "Mistral Instruct v0.3 7B", }, "model_route_id": "mistralai__mistral-7b-instruct-v0-3", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.716, }, }, "score": 0.716, "score_details": { "score": 0.716, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-7b-instruct-v0-3/helm_lite_mistralai_mistral_7b_instruct_v0_3_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-14b", "name": "Qwen1.5 14B", }, "model_route_id": "qwen__qwen1-5-14b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.711, }, }, "score": 0.711, "score_details": { "score": 0.711, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen1-5-14b/helm_lite_qwen_qwen1_5_14b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "writer", "id": "writer/palmyra-x-v3", "name": "Palmyra X V3 72B", }, "model_route_id": "writer__palmyra-x-v3", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.706, }, }, "score": 0.706, "score_details": { "score": 0.706, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-x-v3/helm_lite_writer_palmyra_x_v3_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "microsoft", "id": "microsoft/phi-2", "name": "Phi-2", }, "model_route_id": "microsoft__phi-2", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.703, }, }, "score": 0.703, "score_details": { "score": 0.703, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/microsoft__phi-2/helm_lite_microsoft_phi_2_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "01-ai", "id": "01-ai/yi-6b", "name": "Yi 6B", }, "model_route_id": "01-ai__yi-6b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.702, }, }, "score": 0.702, "score_details": { "score": 0.702, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/01-ai__yi-6b/helm_lite_01_ai_yi_6b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "meta", "id": "meta/llama-2-7b", "name": "Llama 2 7B", }, "model_route_id": "meta__llama-2-7b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.686, }, }, "score": 0.686, "score_details": { "score": 0.686, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-2-7b/helm_lite_meta_llama_2_7b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "AlephAlpha", "id": "alephalpha/luminous-extended", "name": "Luminous Extended 30B", }, "model_route_id": "alephalpha__luminous-extended", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.684, }, }, "score": 0.684, "score_details": { "score": 0.684, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alephalpha__luminous-extended/helm_lite_alephalpha_luminous_extended_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-2-1", "name": "Claude 2.1", }, "model_route_id": "anthropic__claude-2-1", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.677, }, }, "score": 0.677, "score_details": { "score": 0.677, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-2-1/helm_lite_anthropic_claude_2_1_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-40b", "name": "Falcon 40B", }, "model_route_id": "tiiuae__falcon-40b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.671, }, }, "score": 0.671, "score_details": { "score": 0.671, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/tiiuae__falcon-40b/helm_lite_tiiuae_falcon_40b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "ai21", "id": "ai21/jamba-1-5-large", "name": "Jamba 1.5 Large", }, "model_route_id": "ai21__jamba-1-5-large", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.664, }, }, "score": 0.664, "score_details": { "score": 0.664, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__jamba-1-5-large/helm_lite_ai21_jamba_1_5_large_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "ai21", "id": "ai21/jamba-instruct", "name": "Jamba Instruct", }, "model_route_id": "ai21__jamba-instruct", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.658, }, }, "score": 0.658, "score_details": { "score": 0.658, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ai21__jamba-instruct/helm_lite_ai21_jamba_instruct_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0613", "name": "GPT-3.5 Turbo 0613", }, "model_route_id": "openai__gpt-3-5-turbo-0613", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.655, }, }, "score": 0.655, "score_details": { "score": 0.655, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-3-5-turbo-0613/helm_lite_openai_gpt_3_5_turbo_0613_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "snowflake", "id": "snowflake/snowflake-arctic-instruct", "name": "Arctic Instruct", }, "model_route_id": "snowflake__snowflake-arctic-instruct", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.654, }, }, "score": 0.654, "score_details": { "score": 0.654, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/snowflake__snowflake-arctic-instruct/helm_lite_snowflake_snowflake_arctic_instruct_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "AlephAlpha", "id": "alephalpha/luminous-base", "name": "Luminous Base 13B", }, "model_route_id": "alephalpha__luminous-base", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.633, }, }, "score": 0.633, "score_details": { "score": 0.633, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/alephalpha__luminous-base/helm_lite_alephalpha_luminous_base_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "cohere", "id": "cohere/command-light", "name": "Command Light", }, "model_route_id": "cohere__command-light", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.629, }, }, "score": 0.629, "score_details": { "score": 0.629, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__command-light/helm_lite_cohere_command_light_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "tiiuae", "id": "tiiuae/falcon-7b", "name": "Falcon 7B", }, "model_route_id": "tiiuae__falcon-7b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.621, }, }, "score": 0.621, "score_details": { "score": 0.621, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/tiiuae__falcon-7b/helm_lite_tiiuae_falcon_7b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-instant-1-2", "name": "Claude Instant 1.2", }, "model_route_id": "anthropic__claude-instant-1-2", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.616, }, }, "score": 0.616, "score_details": { "score": 0.616, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-instant-1-2/helm_lite_anthropic_claude_instant_1_2_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-72b", "name": "Qwen1.5 72B", }, "model_route_id": "qwen__qwen1-5-72b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.601, }, }, "score": 0.601, "score_details": { "score": 0.601, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen1-5-72b/helm_lite_qwen_qwen1_5_72b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "allenai", "id": "allenai/olmo-7b", "name": "OLMo 7B", }, "model_route_id": "allenai__olmo-7b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.597, }, }, "score": 0.597, "score_details": { "score": 0.597, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/allenai__olmo-7b/helm_lite_allenai_olmo_7b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-32b", "name": "Qwen1.5 32B", }, "model_route_id": "qwen__qwen1-5-32b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.589, }, }, "score": 0.589, "score_details": { "score": 0.589, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen1-5-32b/helm_lite_qwen_qwen1_5_32b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "google", "id": "google/text-unicorn-001", "name": "PaLM-2 Unicorn", }, "model_route_id": "google__text-unicorn-001", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.583, }, }, "score": 0.583, "score_details": { "score": 0.583, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__text-unicorn-001/helm_lite_google_text_unicorn_001_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-llm-67b-chat", "name": "DeepSeek LLM Chat 67B", }, "model_route_id": "deepseek-ai__deepseek-llm-67b-chat", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.581, }, }, "score": 0.581, "score_details": { "score": 0.581, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek-ai__deepseek-llm-67b-chat/helm_lite_deepseek_ai_deepseek_llm_67b_chat_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-small-2402", "name": "Mistral Small 2402", }, "model_route_id": "mistralai__mistral-small-2402", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.519, }, }, "score": 0.519, "score_details": { "score": 0.519, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-small-2402/helm_lite_mistralai_mistral_small_2402_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "databricks", "id": "databricks/dbrx-instruct", "name": "DBRX Instruct", }, "model_route_id": "databricks__dbrx-instruct", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.488, }, }, "score": 0.488, "score_details": { "score": 0.488, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/databricks__dbrx-instruct/helm_lite_databricks_dbrx_instruct_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-large-2402", "name": "Mistral Large 2402", }, "model_route_id": "mistralai__mistral-large-2402", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.454, }, }, "score": 0.454, "score_details": { "score": 0.454, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-large-2402/helm_lite_mistralai_mistral_large_2402_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-medium-2312", "name": "Mistral Medium 2312", }, "model_route_id": "mistralai__mistral-medium-2312", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.449, }, }, "score": 0.449, "score_details": { "score": 0.449, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-medium-2312/helm_lite_mistralai_mistral_medium_2312_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-7b", "name": "Qwen1.5 7B", }, "model_route_id": "qwen__qwen1-5-7b", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.448, }, }, "score": 0.448, "score_details": { "score": 0.448, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen1-5-7b/helm_lite_qwen_qwen1_5_7b_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "01-ai", "id": "01-ai/yi-large-preview", "name": "Yi Large Preview", }, "model_route_id": "01-ai__yi-large-preview", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.373, }, }, "score": 0.373, "score_details": { "score": 0.373, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/01-ai__yi-large-preview/helm_lite_01_ai_yi_large_preview_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-opus", "name": "Claude 3 Opus 20240229", }, "model_route_id": "anthropic__claude-3-opus", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.351, }, }, "score": 0.351, "score_details": { "score": 0.351, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-opus/helm_lite_anthropic_claude_3_opus_20240229_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-haiku", "name": "Claude 3 Haiku 20240307", }, "model_route_id": "anthropic__claude-3-haiku", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.244, }, }, "score": 0.244, "score_details": { "score": 0.244, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-haiku/helm_lite_anthropic_claude_3_haiku_20240307_1774096306_427425.json", }, { "evaluation_timestamp": "1774096306.427425", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-sonnet", "name": "Claude 3 Sonnet 20240229", }, "model_route_id": "anthropic__claude-3-sonnet", "result": { "canonical_display_name": "NarrativeQA / F1", "detailed_evaluation_results_url": undefined, "display_name": "NarrativeQA / F1", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "F1", "evaluation_timestamp": "1774096306.427425", "metric_config": { "evaluation_description": "F1 on NarrativeQA", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": "proportion", }, "metric_key": "f1", "metric_summary_id": "helm_lite_narrativeqa_f1", "score_details": { "score": 0.111, }, }, "score": 0.111, "score_details": { "score": 0.111, }, "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_lite", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-sonnet/helm_lite_anthropic_claude_3_sonnet_20240229_1774096306_427425.json", }, ], "models_count": 91, "provenance_summary": { "first_party_only_groups": 0, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 0, "third_party": 91, "unspecified": 0, }, "total_groups": 89, "total_results": 91, }, "reproducibility_summary": { "has_reproducibility_gap_count": 91, "populated_ratio_avg": 0, "results_total": 91, }, "root_metrics": [ { "canonical_display_name": "NarrativeQA / F1", "display_name": "NarrativeQA / F1", "lower_is_better": false, "metric_key": "f1", "metric_name": "F1", "metric_summary_id": "helm_lite_narrativeqa_f1", "models_count": 91, "top_score": 0.804, "unit": "proportion", }, ], "source_data": { "dataset_name": "NarrativeQA", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json", ], }, "source_types": [], "subtasks": [], "third_party_ratio": 0, "worst_model": { "name": "Claude 3 Sonnet 20240229", "score": 0.111, }, } `; exports[`hfEvalDetailToSummary > helm_safety_simplesafetytests — Safety regression-bait — pipeline category=general for an obvious safety eval 1`] = ` { "avg_score": 0.9600919540229885, "avg_score_norm": 0.9600919540229885, "benchmark_card": undefined, "best_model": { "name": "Palmyra X5", "score": 1, }, "canonical_display_name": "SimpleSafetyTests", "category": "Safety", "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 0, "total_groups": 85, "variant_divergent_count": 0, }, "composite_benchmark_key": "SimpleSafetyTests", "composite_benchmark_name": "SimpleSafetyTests", "evalcards": { "annotations": { "benchmark_comparability": { "cross_party_divergence_groups": [], "variant_divergence_groups": [], }, "reporting_completeness": { "completeness_score": 0.10714285714285714, "field_scores": [ { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.name", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.overview", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.data_type", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.domains", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.languages", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.similar_benchmarks", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.benchmark_details.resources", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.goal", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.audience", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.tasks", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.limitations", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.methods", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.metrics", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.calculation", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.interpretation", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.baseline_results", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.methodology.validation", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "score": 0, }, { "coverage_type": "full", "field_path": "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "score": 0, }, { "coverage_type": "partial", "field_path": "autobenchmarkcard.data", "score": 0, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_type", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.source_organization_name", "score": 1, }, { "coverage_type": "full", "field_path": "eee_eval.source_metadata.evaluator_relationship", "score": 1, }, { "coverage_type": "reserved", "field_path": "evalcards.lifecycle_status", "score": 0, }, { "coverage_type": "reserved", "field_path": "evalcards.preregistration_url", "score": 0, }, ], "missing_required_fields": [ "autobenchmarkcard.benchmark_details.name", "autobenchmarkcard.benchmark_details.overview", "autobenchmarkcard.benchmark_details.data_type", "autobenchmarkcard.benchmark_details.domains", "autobenchmarkcard.benchmark_details.languages", "autobenchmarkcard.benchmark_details.similar_benchmarks", "autobenchmarkcard.benchmark_details.resources", "autobenchmarkcard.purpose_and_intended_users.goal", "autobenchmarkcard.purpose_and_intended_users.audience", "autobenchmarkcard.purpose_and_intended_users.tasks", "autobenchmarkcard.purpose_and_intended_users.limitations", "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "autobenchmarkcard.methodology.methods", "autobenchmarkcard.methodology.metrics", "autobenchmarkcard.methodology.calculation", "autobenchmarkcard.methodology.interpretation", "autobenchmarkcard.methodology.baseline_results", "autobenchmarkcard.methodology.validation", "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "autobenchmarkcard.data", "evalcards.lifecycle_status", "evalcards.preregistration_url", ], "partial_fields": [], "signal_version": "1.0", "total_fields_evaluated": 28, }, }, }, "evaluation_id": "helm_safety_simplesafetytests", "evaluation_name": "SimpleSafetyTests", "evaluator_names": [], "latest_source_name": "SimpleSafetyTests", "leaderboard_metrics": [ { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "column_key": "root:helm_safety_simplesafetytests_lm_evaluated_safety_score", "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "lower_is_better": false, "metric_name": "LM Evaluated Safety score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "scope": "root", "subtask_key": undefined, "subtask_name": undefined, "unit": undefined, }, ], "leaderboard_rows": [ { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-x5", "name": "Palmyra X5", }, "model_route_id": "writer__palmyra-x5", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-x-004", "name": "Palmyra-X-004", }, "model_route_id": "writer__palmyra-x-004", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-fin", "name": "Palmyra Fin", }, "model_route_id": "writer__palmyra-fin", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen3-235b-a22b-instruct-2507-fp8", "name": "Qwen3 235B A22B Instruct 2507 FP8", }, "model_route_id": "qwen__qwen3-235b-a22b-instruct-2507-fp8", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-72b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 72B", }, "model_route_id": "qwen__qwen2-5-72b-instruct-turbo", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o4-mini", "name": "o4-mini 2025-04-16", }, "model_route_id": "openai__o4-mini", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-oss-20b", "name": "gpt-oss-20b", }, "model_route_id": "openai__gpt-oss-20b", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-oss-120b", "name": "gpt-oss-120b", }, "model_route_id": "openai__gpt-oss-120b", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-nano", "name": "GPT-5 nano 2025-08-07", }, "model_route_id": "openai__gpt-5-nano", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-mini", "name": "GPT-5 mini 2025-08-07", }, "model_route_id": "openai__gpt-5-mini", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-5-preview", "name": "GPT-4.5 2025-02-27 preview", }, "model_route_id": "openai__gpt-4-5-preview", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1-mini", "name": "GPT-4.1 mini 2025-04-14", }, "model_route_id": "openai__gpt-4-1-mini", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1", "name": "GPT-4.1 2025-04-14", }, "model_route_id": "openai__gpt-4-1", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "moonshotai", "id": "moonshotai/kimi-k2-instruct", "name": "Kimi K2 Instruct", }, "model_route_id": "moonshotai__kimi-k2-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-micro-with-guardian", "name": "IBM Granite 4.0 Micro with guardian", }, "model_route_id": "ibm__granite-4-0-micro-with-guardian", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-h-small-with-guardian", "name": "IBM Granite 4.0 Small with guardian", }, "model_route_id": "ibm__granite-4-0-h-small-with-guardian", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/command-r-plus", "name": "Command R Plus", }, "model_route_id": "cohere__command-r-plus", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4-5", "name": "Claude 4.5 Sonnet 20250929", }, "model_route_id": "anthropic__claude-sonnet-4-5", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4", "name": "Claude 4 Sonnet 20250514, extended thinking", }, "model_route_id": "anthropic__claude-sonnet-4", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.995, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-opus-4", "name": "Claude 4 Opus 20250514, extended thinking", }, "model_route_id": "anthropic__claude-opus-4", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.995, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-sonnet", "name": "Claude 3 Sonnet 20240229", }, "model_route_id": "anthropic__claude-3-sonnet", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-opus", "name": "Claude 3 Opus 20240229", }, "model_route_id": "anthropic__claude-3-opus", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-haiku", "name": "Claude 3 Haiku 20240307", }, "model_route_id": "anthropic__claude-3-haiku", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-7-sonnet", "name": "Claude 3.7 Sonnet 20250219", }, "model_route_id": "anthropic__claude-3-7-sonnet", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-sonnet", "name": "Claude 3.5 Sonnet 20240620", }, "model_route_id": "anthropic__claude-3-5-sonnet", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 1, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5-1", "name": "GPT-5.1 2025-11-13", }, "model_route_id": "openai__gpt-5-1", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.998, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-5", "name": "GPT-5 2025-08-07", }, "model_route_id": "openai__gpt-5", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.998, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen3-next-80b-a3b-thinking", "name": "Qwen3-Next 80B A3B Thinking", }, "model_route_id": "qwen__qwen3-next-80b-a3b-thinking", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.995, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-3-mini-beta", "name": "Grok 3 mini Beta", }, "model_route_id": "xai__grok-3-mini-beta", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.993, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-4-maverick-17b-128e-instruct-fp8", "name": "Llama 4 Maverick 17Bx128E Instruct FP8", }, "model_route_id": "meta__llama-4-maverick-17b-128e-instruct-fp8", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.993, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-8b-chat", "name": "Llama 3 Instruct 8B", }, "model_route_id": "meta__llama-3-8b-chat", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.993, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "zai-org", "id": "zai-org/glm-4-5-air-fp8", "name": "GLM-4.5-Air-FP8", }, "model_route_id": "zai-org__glm-4-5-air-fp8", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.99, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-72b-chat", "name": "Qwen1.5 Chat 72B", }, "model_route_id": "qwen__qwen1-5-72b-chat", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.99, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o3-mini", "name": "o3-mini 2025-01-31", }, "model_route_id": "openai__o3-mini", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.99, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o3", "name": "o3 2025-04-16", }, "model_route_id": "openai__o3", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.99, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o1", "name": "o1 2024-12-17", }, "model_route_id": "openai__o1", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.99, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-turbo", "name": "GPT-4 Turbo 2024-04-09", }, "model_route_id": "openai__gpt-4-turbo", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.99, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4-1-nano", "name": "GPT-4.1 nano 2025-04-14", }, "model_route_id": "openai__gpt-4-1-nano", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.99, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-70b-chat", "name": "Llama 3 Instruct 70B", }, "model_route_id": "meta__llama-3-70b-chat", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.99, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-8b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 8B", }, "model_route_id": "meta__llama-3-1-8b-instruct-turbo", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.988, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-405b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 405B", }, "model_route_id": "meta__llama-3-1-405b-instruct-turbo", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.988, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "anthropic", "id": "anthropic/claude-haiku-4-5", "name": "Claude 4.5 Haiku 20251001", }, "model_route_id": "anthropic__claude-haiku-4-5", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.988, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "writer", "id": "writer/palmyra-med", "name": "Palmyra Med", }, "model_route_id": "writer__palmyra-med", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.985, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen3-235b-a22b-fp8-tput", "name": "Qwen3 235B A22B FP8 Throughput", }, "model_route_id": "qwen__qwen3-235b-a22b-fp8-tput", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.985, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen2-72b-instruct", "name": "Qwen2 Instruct 72B", }, "model_route_id": "qwen__qwen2-72b-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.985, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o 2024-05-13", }, "model_route_id": "openai__gpt-4o", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.985, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-001", "name": "Gemini 2.0 Flash", }, "model_route_id": "google__gemini-2-0-flash-001", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.985, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-r1-0528", "name": "DeepSeek-R1-0528", }, "model_route_id": "deepseek-ai__deepseek-r1-0528", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.983, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-h-small", "name": "IBM Granite 4.0 Small", }, "model_route_id": "ibm__granite-4-0-h-small", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.98, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-preview-04-17", "name": "Gemini 2.5 Flash 04-17 preview", }, "model_route_id": "google__gemini-2-5-flash-preview-04-17", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.98, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-r1-hide-reasoning", "name": "DeepSeek R1 hide reasoning", }, "model_route_id": "deepseek-ai__deepseek-r1-hide-reasoning", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.98, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "allenai", "id": "allenai/olmo-2-0325-32b-instruct", "name": "OLMo 2 32B Instruct March 2025", }, "model_route_id": "allenai__olmo-2-0325-32b-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.98, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-4o-mini", "name": "GPT-4o mini 2024-07-18", }, "model_route_id": "openai__gpt-4o-mini", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.978, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-lite-preview-02-05", "name": "Gemini 2.0 Flash Lite 02-05 preview", }, "model_route_id": "google__gemini-2-0-flash-lite-preview-02-05", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.977, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "ibm", "id": "ibm/granite-3-3-8b-instruct", "name": "IBM Granite 3.3 8B Instruct", }, "model_route_id": "ibm__granite-3-3-8b-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.975, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-3-pro-preview", "name": "Gemini 3 Pro Preview", }, "model_route_id": "google__gemini-3-pro-preview", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.975, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-0-pro-exp-02-05", "name": "Gemini 2.0 Pro 02-05 preview", }, "model_route_id": "google__gemini-2-0-pro-exp-02-05", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.975, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-001", "name": "Gemini 1.5 Pro 001", }, "model_route_id": "google__gemini-1-5-pro-001", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.975, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-r1", "name": "DeepSeek R1", }, "model_route_id": "deepseek-ai__deepseek-r1", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.975, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/o1-mini", "name": "o1-mini 2024-09-12", }, "model_route_id": "openai__o1-mini", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.97, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-4-scout-17b-16e-instruct", "name": "Llama 4 Scout 17Bx16E Instruct", }, "model_route_id": "meta__llama-4-scout-17b-16e-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.97, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-pro-preview-03-25", "name": "Gemini 2.5 Pro 03-25 preview", }, "model_route_id": "google__gemini-2-5-pro-preview-03-25", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.97, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-001", "name": "Gemini 1.5 Flash 001", }, "model_route_id": "google__gemini-1-5-flash-001", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.97, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-3-beta", "name": "Grok 3 Beta", }, "model_route_id": "xai__grok-3-beta", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.968, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-llm-67b-chat", "name": "DeepSeek LLM Chat 67B", }, "model_route_id": "deepseek-ai__deepseek-llm-67b-chat", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.968, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-lite", "name": "Gemini 2.5 Flash-Lite", }, "model_route_id": "google__gemini-2-5-flash-lite", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.965, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-7b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 7B", }, "model_route_id": "qwen__qwen2-5-7b-instruct-turbo", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.96, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0613", "name": "GPT-3.5 Turbo 0613", }, "model_route_id": "openai__gpt-3-5-turbo-0613", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.958, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "marin-community", "id": "marin-community/marin-8b-instruct", "name": "Marin 8B Instruct", }, "model_route_id": "marin-community__marin-8b-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.958, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-v3", "name": "DeepSeek v3", }, "model_route_id": "deepseek-ai__deepseek-v3", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.953, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-micro", "name": "IBM Granite 4.0 Micro", }, "model_route_id": "ibm__granite-4-0-micro", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.945, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "cohere", "id": "cohere/command-r", "name": "Command R", }, "model_route_id": "cohere__command-r", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.943, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-small-2501", "name": "Mistral Small 3 2501", }, "model_route_id": "mistralai__mistral-small-2501", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.932, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "meta", "id": "meta/llama-3-1-70b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 70B", }, "model_route_id": "meta__llama-3-1-70b-instruct-turbo", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.925, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "xai", "id": "xai/grok-4-0709", "name": "Grok 4 0709", }, "model_route_id": "xai__grok-4-0709", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.922, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-1106", "name": "GPT-3.5 Turbo 1106", }, "model_route_id": "openai__gpt-3-5-turbo-1106", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.922, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0125", "name": "GPT-3.5 Turbo 0125", }, "model_route_id": "openai__gpt-3-5-turbo-0125", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.92, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x22b-instruct-v0-1", "name": "Mixtral Instruct 8x22B", }, "model_route_id": "mistralai__mixtral-8x22b-instruct-v0-1", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.915, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x7b-instruct-v0-1", "name": "Mixtral Instruct 8x7B", }, "model_route_id": "mistralai__mixtral-8x7b-instruct-v0-1", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.905, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-instruct-v0-3", "name": "Mistral Instruct v0.3 7B", }, "model_route_id": "mistralai__mistral-7b-instruct-v0-3", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.81, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "allenai", "id": "allenai/olmo-2-1124-13b-instruct", "name": "OLMo 2 13B Instruct November 2024", }, "model_route_id": "allenai__olmo-2-1124-13b-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.81, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "allenai", "id": "allenai/olmo-2-1124-7b-instruct", "name": "OLMo 2 7B Instruct November 2024", }, "model_route_id": "allenai__olmo-2-1124-7b-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.775, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "allenai", "id": "allenai/olmoe-1b-7b-0125-instruct", "name": "OLMoE 1B-7B Instruct January 2025", }, "model_route_id": "allenai__olmoe-1b-7b-0125-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.725, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "databricks", "id": "databricks/dbrx-instruct", "name": "DBRX Instruct", }, "model_route_id": "databricks__dbrx-instruct", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.535, }, }, { "annotations_by_metric": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_timestamp": "1777076383.2276576", "metrics_present": 1, "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-instruct-v0-1", "name": "Mistral Instruct v0.1 7B", }, "model_route_id": "mistralai__mistral-7b-instruct-v0-1", "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "values": { "root:helm_safety_simplesafetytests_lm_evaluated_safety_score": 0.432, }, }, ], "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_names": [ "LM Evaluated Safety score", ], "metrics_count": 1, "missing_generation_config_count": 0, "model_results": [ { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "writer", "id": "writer/palmyra-x5", "name": "Palmyra X5", }, "model_route_id": "writer__palmyra-x5", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-x5/helm_safety_writer_palmyra_x5_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "writer", "id": "writer/palmyra-x-004", "name": "Palmyra-X-004", }, "model_route_id": "writer__palmyra-x-004", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-x-004/helm_safety_writer_palmyra_x_004_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "writer", "id": "writer/palmyra-fin", "name": "Palmyra Fin", }, "model_route_id": "writer__palmyra-fin", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-fin/helm_safety_writer_palmyra_fin_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "qwen", "id": "qwen/qwen3-235b-a22b-instruct-2507-fp8", "name": "Qwen3 235B A22B Instruct 2507 FP8", }, "model_route_id": "qwen__qwen3-235b-a22b-instruct-2507-fp8", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen3-235b-a22b-instruct-2507-fp8/helm_safety_qwen_qwen3_235b_a22b_instruct_2507_fp8_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-72b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 72B", }, "model_route_id": "qwen__qwen2-5-72b-instruct-turbo", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen2-5-72b-instruct-turbo/helm_safety_qwen_qwen2_5_72b_instruct_turbo_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/o4-mini", "name": "o4-mini 2025-04-16", }, "model_route_id": "openai__o4-mini", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o4-mini/helm_safety_openai_o4_mini_2025_04_16_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-oss-20b", "name": "gpt-oss-20b", }, "model_route_id": "openai__gpt-oss-20b", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-oss-20b/helm_safety_openai_gpt_oss_20b_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-oss-120b", "name": "gpt-oss-120b", }, "model_route_id": "openai__gpt-oss-120b", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-oss-120b/helm_safety_openai_gpt_oss_120b_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-5-nano", "name": "GPT-5 nano 2025-08-07", }, "model_route_id": "openai__gpt-5-nano", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-nano/helm_safety_openai_gpt_5_nano_2025_08_07_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-5-mini", "name": "GPT-5 mini 2025-08-07", }, "model_route_id": "openai__gpt-5-mini", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-mini/helm_safety_openai_gpt_5_mini_2025_08_07_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-4-5-preview", "name": "GPT-4.5 2025-02-27 preview", }, "model_route_id": "openai__gpt-4-5-preview", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-5-preview/helm_safety_openai_gpt_4_5_preview_2025_02_27_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-4-1-mini", "name": "GPT-4.1 mini 2025-04-14", }, "model_route_id": "openai__gpt-4-1-mini", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1-mini/helm_safety_openai_gpt_4_1_mini_2025_04_14_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-4-1", "name": "GPT-4.1 2025-04-14", }, "model_route_id": "openai__gpt-4-1", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1/helm_safety_openai_gpt_4_1_2025_04_14_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "moonshotai", "id": "moonshotai/kimi-k2-instruct", "name": "Kimi K2 Instruct", }, "model_route_id": "moonshotai__kimi-k2-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/moonshotai__kimi-k2-instruct/helm_safety_moonshotai_kimi_k2_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-micro-with-guardian", "name": "IBM Granite 4.0 Micro with guardian", }, "model_route_id": "ibm__granite-4-0-micro-with-guardian", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ibm__granite-4-0-micro-with-guardian/helm_safety_ibm_granite_4_0_micro_with_guardian_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-h-small-with-guardian", "name": "IBM Granite 4.0 Small with guardian", }, "model_route_id": "ibm__granite-4-0-h-small-with-guardian", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ibm__granite-4-0-h-small-with-guardian/helm_safety_ibm_granite_4_0_h_small_with_guardian_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "cohere", "id": "cohere/command-r-plus", "name": "Command R Plus", }, "model_route_id": "cohere__command-r-plus", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__command-r-plus/helm_safety_cohere_command_r_plus_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4-5", "name": "Claude 4.5 Sonnet 20250929", }, "model_route_id": "anthropic__claude-sonnet-4-5", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-sonnet-4-5/helm_safety_anthropic_claude_sonnet_4_5_20250929_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4", "name": "Claude 4 Sonnet 20250514, extended thinking", }, "model_route_id": "anthropic__claude-sonnet-4", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-sonnet-4/helm_safety_anthropic_claude_sonnet_4_20250514_thinking_10k_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-opus-4", "name": "Claude 4 Opus 20250514, extended thinking", }, "model_route_id": "anthropic__claude-opus-4", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-opus-4/helm_safety_anthropic_claude_opus_4_20250514_thinking_10k_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-sonnet", "name": "Claude 3 Sonnet 20240229", }, "model_route_id": "anthropic__claude-3-sonnet", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-sonnet/helm_safety_anthropic_claude_3_sonnet_20240229_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-opus", "name": "Claude 3 Opus 20240229", }, "model_route_id": "anthropic__claude-3-opus", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-opus/helm_safety_anthropic_claude_3_opus_20240229_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-haiku", "name": "Claude 3 Haiku 20240307", }, "model_route_id": "anthropic__claude-3-haiku", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-haiku/helm_safety_anthropic_claude_3_haiku_20240307_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-7-sonnet", "name": "Claude 3.7 Sonnet 20250219", }, "model_route_id": "anthropic__claude-3-7-sonnet", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-7-sonnet/helm_safety_anthropic_claude_3_7_sonnet_20250219_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-3-5-sonnet", "name": "Claude 3.5 Sonnet 20240620", }, "model_route_id": "anthropic__claude-3-5-sonnet", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 1, }, }, "score": 1, "score_details": { "score": 1, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-3-5-sonnet/helm_safety_anthropic_claude_3_5_sonnet_20240620_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-5-1", "name": "GPT-5.1 2025-11-13", }, "model_route_id": "openai__gpt-5-1", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.998, }, }, "score": 0.998, "score_details": { "score": 0.998, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-1/helm_safety_openai_gpt_5_1_2025_11_13_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-5", "name": "GPT-5 2025-08-07", }, "model_route_id": "openai__gpt-5", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.998, }, }, "score": 0.998, "score_details": { "score": 0.998, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5/helm_safety_openai_gpt_5_2025_08_07_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "qwen", "id": "qwen/qwen3-next-80b-a3b-thinking", "name": "Qwen3-Next 80B A3B Thinking", }, "model_route_id": "qwen__qwen3-next-80b-a3b-thinking", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.995, }, }, "score": 0.995, "score_details": { "score": 0.995, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen3-next-80b-a3b-thinking/helm_safety_qwen_qwen3_next_80b_a3b_thinking_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-sonnet-4", "name": "Claude 4 Sonnet 20250514", }, "model_route_id": "anthropic__claude-sonnet-4", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.995, }, }, "score": 0.995, "score_details": { "score": 0.995, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-sonnet-4/helm_safety_anthropic_claude_sonnet_4_20250514_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-opus-4", "name": "Claude 4 Opus 20250514", }, "model_route_id": "anthropic__claude-opus-4", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.995, }, }, "score": 0.995, "score_details": { "score": 0.995, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-opus-4/helm_safety_anthropic_claude_opus_4_20250514_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "xai", "id": "xai/grok-3-mini-beta", "name": "Grok 3 mini Beta", }, "model_route_id": "xai__grok-3-mini-beta", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.993, }, }, "score": 0.993, "score_details": { "score": 0.993, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-3-mini-beta/helm_safety_xai_grok_3_mini_beta_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "meta", "id": "meta/llama-4-maverick-17b-128e-instruct-fp8", "name": "Llama 4 Maverick 17Bx128E Instruct FP8", }, "model_route_id": "meta__llama-4-maverick-17b-128e-instruct-fp8", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.993, }, }, "score": 0.993, "score_details": { "score": 0.993, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-4-maverick-17b-128e-instruct-fp8/helm_safety_meta_llama_4_maverick_17b_128e_instruct_fp8_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "meta", "id": "meta/llama-3-8b-chat", "name": "Llama 3 Instruct 8B", }, "model_route_id": "meta__llama-3-8b-chat", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.993, }, }, "score": 0.993, "score_details": { "score": 0.993, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-8b-chat/helm_safety_meta_llama_3_8b_chat_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "zai-org", "id": "zai-org/glm-4-5-air-fp8", "name": "GLM-4.5-Air-FP8", }, "model_route_id": "zai-org__glm-4-5-air-fp8", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.99, }, }, "score": 0.99, "score_details": { "score": 0.99, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/zai-org__glm-4-5-air-fp8/helm_safety_zai_org_glm_4_5_air_fp8_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "qwen", "id": "qwen/qwen1-5-72b-chat", "name": "Qwen1.5 Chat 72B", }, "model_route_id": "qwen__qwen1-5-72b-chat", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.99, }, }, "score": 0.99, "score_details": { "score": 0.99, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen1-5-72b-chat/helm_safety_qwen_qwen1_5_72b_chat_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/o3-mini", "name": "o3-mini 2025-01-31", }, "model_route_id": "openai__o3-mini", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.99, }, }, "score": 0.99, "score_details": { "score": 0.99, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o3-mini/helm_safety_openai_o3_mini_2025_01_31_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/o3", "name": "o3 2025-04-16", }, "model_route_id": "openai__o3", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.99, }, }, "score": 0.99, "score_details": { "score": 0.99, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o3/helm_safety_openai_o3_2025_04_16_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/o1", "name": "o1 2024-12-17", }, "model_route_id": "openai__o1", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.99, }, }, "score": 0.99, "score_details": { "score": 0.99, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o1/helm_safety_openai_o1_2024_12_17_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-4-turbo", "name": "GPT-4 Turbo 2024-04-09", }, "model_route_id": "openai__gpt-4-turbo", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.99, }, }, "score": 0.99, "score_details": { "score": 0.99, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-turbo/helm_safety_openai_gpt_4_turbo_2024_04_09_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-4-1-nano", "name": "GPT-4.1 nano 2025-04-14", }, "model_route_id": "openai__gpt-4-1-nano", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.99, }, }, "score": 0.99, "score_details": { "score": 0.99, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4-1-nano/helm_safety_openai_gpt_4_1_nano_2025_04_14_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "meta", "id": "meta/llama-3-70b-chat", "name": "Llama 3 Instruct 70B", }, "model_route_id": "meta__llama-3-70b-chat", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.99, }, }, "score": 0.99, "score_details": { "score": 0.99, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-70b-chat/helm_safety_meta_llama_3_70b_chat_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "meta", "id": "meta/llama-3-1-8b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 8B", }, "model_route_id": "meta__llama-3-1-8b-instruct-turbo", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.988, }, }, "score": 0.988, "score_details": { "score": 0.988, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-8b-instruct-turbo/helm_safety_meta_llama_3_1_8b_instruct_turbo_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "meta", "id": "meta/llama-3-1-405b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 405B", }, "model_route_id": "meta__llama-3-1-405b-instruct-turbo", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.988, }, }, "score": 0.988, "score_details": { "score": 0.988, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-405b-instruct-turbo/helm_safety_meta_llama_3_1_405b_instruct_turbo_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "anthropic", "id": "anthropic/claude-haiku-4-5", "name": "Claude 4.5 Haiku 20251001", }, "model_route_id": "anthropic__claude-haiku-4-5", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.988, }, }, "score": 0.988, "score_details": { "score": 0.988, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__claude-haiku-4-5/helm_safety_anthropic_claude_haiku_4_5_20251001_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "writer", "id": "writer/palmyra-med", "name": "Palmyra Med", }, "model_route_id": "writer__palmyra-med", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.985, }, }, "score": 0.985, "score_details": { "score": 0.985, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/writer__palmyra-med/helm_safety_writer_palmyra_med_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "qwen", "id": "qwen/qwen3-235b-a22b-fp8-tput", "name": "Qwen3 235B A22B FP8 Throughput", }, "model_route_id": "qwen__qwen3-235b-a22b-fp8-tput", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.985, }, }, "score": 0.985, "score_details": { "score": 0.985, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen3-235b-a22b-fp8-tput/helm_safety_qwen_qwen3_235b_a22b_fp8_tput_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "qwen", "id": "qwen/qwen2-72b-instruct", "name": "Qwen2 Instruct 72B", }, "model_route_id": "qwen__qwen2-72b-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.985, }, }, "score": 0.985, "score_details": { "score": 0.985, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen2-72b-instruct/helm_safety_qwen_qwen2_72b_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-4o", "name": "GPT-4o 2024-05-13", }, "model_route_id": "openai__gpt-4o", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.985, }, }, "score": 0.985, "score_details": { "score": 0.985, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o/helm_safety_openai_gpt_4o_2024_05_13_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-001", "name": "Gemini 2.0 Flash", }, "model_route_id": "google__gemini-2-0-flash-001", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.985, }, }, "score": 0.985, "score_details": { "score": 0.985, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash-001/helm_safety_google_gemini_2_0_flash_001_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-r1-0528", "name": "DeepSeek-R1-0528", }, "model_route_id": "deepseek-ai__deepseek-r1-0528", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.983, }, }, "score": 0.983, "score_details": { "score": 0.983, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek-ai__deepseek-r1-0528/helm_safety_deepseek_ai_deepseek_r1_0528_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-h-small", "name": "IBM Granite 4.0 Small", }, "model_route_id": "ibm__granite-4-0-h-small", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.98, }, }, "score": 0.98, "score_details": { "score": 0.98, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ibm__granite-4-0-h-small/helm_safety_ibm_granite_4_0_h_small_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-preview-04-17", "name": "Gemini 2.5 Flash 04-17 preview", }, "model_route_id": "google__gemini-2-5-flash-preview-04-17", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.98, }, }, "score": 0.98, "score_details": { "score": 0.98, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash-preview-04-17/helm_safety_google_gemini_2_5_flash_preview_04_17_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-r1-hide-reasoning", "name": "DeepSeek R1 hide reasoning", }, "model_route_id": "deepseek-ai__deepseek-r1-hide-reasoning", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.98, }, }, "score": 0.98, "score_details": { "score": 0.98, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek-ai__deepseek-r1-hide-reasoning/helm_safety_deepseek_ai_deepseek_r1_hide_reasoning_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "allenai", "id": "allenai/olmo-2-0325-32b-instruct", "name": "OLMo 2 32B Instruct March 2025", }, "model_route_id": "allenai__olmo-2-0325-32b-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.98, }, }, "score": 0.98, "score_details": { "score": 0.98, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/allenai__olmo-2-0325-32b-instruct/helm_safety_allenai_olmo_2_0325_32b_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-4o-mini", "name": "GPT-4o mini 2024-07-18", }, "model_route_id": "openai__gpt-4o-mini", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.978, }, }, "score": 0.978, "score_details": { "score": 0.978, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o-mini/helm_safety_openai_gpt_4o_mini_2024_07_18_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "google", "id": "google/gemini-2-0-flash-lite-preview-02-05", "name": "Gemini 2.0 Flash Lite 02-05 preview", }, "model_route_id": "google__gemini-2-0-flash-lite-preview-02-05", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.977, }, }, "score": 0.977, "score_details": { "score": 0.977, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-flash-lite-preview-02-05/helm_safety_google_gemini_2_0_flash_lite_preview_02_05_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "ibm", "id": "ibm/granite-3-3-8b-instruct", "name": "IBM Granite 3.3 8B Instruct", }, "model_route_id": "ibm__granite-3-3-8b-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.975, }, }, "score": 0.975, "score_details": { "score": 0.975, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ibm__granite-3-3-8b-instruct/helm_safety_ibm_granite_3_3_8b_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "google", "id": "google/gemini-3-pro-preview", "name": "Gemini 3 Pro Preview", }, "model_route_id": "google__gemini-3-pro-preview", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.975, }, }, "score": 0.975, "score_details": { "score": 0.975, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro-preview/helm_safety_google_gemini_3_pro_preview_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "google", "id": "google/gemini-2-0-pro-exp-02-05", "name": "Gemini 2.0 Pro 02-05 preview", }, "model_route_id": "google__gemini-2-0-pro-exp-02-05", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.975, }, }, "score": 0.975, "score_details": { "score": 0.975, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-0-pro-exp-02-05/helm_safety_google_gemini_2_0_pro_exp_02_05_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "google", "id": "google/gemini-1-5-pro-001", "name": "Gemini 1.5 Pro 001", }, "model_route_id": "google__gemini-1-5-pro-001", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.975, }, }, "score": 0.975, "score_details": { "score": 0.975, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-pro-001/helm_safety_google_gemini_1_5_pro_001_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-r1", "name": "DeepSeek R1", }, "model_route_id": "deepseek-ai__deepseek-r1", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.975, }, }, "score": 0.975, "score_details": { "score": 0.975, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek-ai__deepseek-r1/helm_safety_deepseek_ai_deepseek_r1_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/o1-mini", "name": "o1-mini 2024-09-12", }, "model_route_id": "openai__o1-mini", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.97, }, }, "score": 0.97, "score_details": { "score": 0.97, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o1-mini/helm_safety_openai_o1_mini_2024_09_12_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "meta", "id": "meta/llama-4-scout-17b-16e-instruct", "name": "Llama 4 Scout 17Bx16E Instruct", }, "model_route_id": "meta__llama-4-scout-17b-16e-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.97, }, }, "score": 0.97, "score_details": { "score": 0.97, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-4-scout-17b-16e-instruct/helm_safety_meta_llama_4_scout_17b_16e_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "google", "id": "google/gemini-2-5-pro-preview-03-25", "name": "Gemini 2.5 Pro 03-25 preview", }, "model_route_id": "google__gemini-2-5-pro-preview-03-25", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.97, }, }, "score": 0.97, "score_details": { "score": 0.97, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-pro-preview-03-25/helm_safety_google_gemini_2_5_pro_preview_03_25_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "google", "id": "google/gemini-1-5-flash-001", "name": "Gemini 1.5 Flash 001", }, "model_route_id": "google__gemini-1-5-flash-001", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.97, }, }, "score": 0.97, "score_details": { "score": 0.97, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-1-5-flash-001/helm_safety_google_gemini_1_5_flash_001_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "xai", "id": "xai/grok-3-beta", "name": "Grok 3 Beta", }, "model_route_id": "xai__grok-3-beta", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.968, }, }, "score": 0.968, "score_details": { "score": 0.968, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-3-beta/helm_safety_xai_grok_3_beta_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-llm-67b-chat", "name": "DeepSeek LLM Chat 67B", }, "model_route_id": "deepseek-ai__deepseek-llm-67b-chat", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.968, }, }, "score": 0.968, "score_details": { "score": 0.968, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek-ai__deepseek-llm-67b-chat/helm_safety_deepseek_ai_deepseek_llm_67b_chat_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "google", "id": "google/gemini-2-5-flash-lite", "name": "Gemini 2.5 Flash-Lite", }, "model_route_id": "google__gemini-2-5-flash-lite", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.965, }, }, "score": 0.965, "score_details": { "score": 0.965, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash-lite/helm_safety_google_gemini_2_5_flash_lite_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "qwen", "id": "qwen/qwen2-5-7b-instruct-turbo", "name": "Qwen2.5 Instruct Turbo 7B", }, "model_route_id": "qwen__qwen2-5-7b-instruct-turbo", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.96, }, }, "score": 0.96, "score_details": { "score": 0.96, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/qwen__qwen2-5-7b-instruct-turbo/helm_safety_qwen_qwen2_5_7b_instruct_turbo_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0613", "name": "GPT-3.5 Turbo 0613", }, "model_route_id": "openai__gpt-3-5-turbo-0613", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.958, }, }, "score": 0.958, "score_details": { "score": 0.958, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-3-5-turbo-0613/helm_safety_openai_gpt_3_5_turbo_0613_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "marin-community", "id": "marin-community/marin-8b-instruct", "name": "Marin 8B Instruct", }, "model_route_id": "marin-community__marin-8b-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.958, }, }, "score": 0.958, "score_details": { "score": 0.958, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/marin-community__marin-8b-instruct/helm_safety_marin_community_marin_8b_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "deepseek-ai", "id": "deepseek-ai/deepseek-v3", "name": "DeepSeek v3", }, "model_route_id": "deepseek-ai__deepseek-v3", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.953, }, }, "score": 0.953, "score_details": { "score": 0.953, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/deepseek-ai__deepseek-v3/helm_safety_deepseek_ai_deepseek_v3_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "ibm", "id": "ibm/granite-4-0-micro", "name": "IBM Granite 4.0 Micro", }, "model_route_id": "ibm__granite-4-0-micro", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.945, }, }, "score": 0.945, "score_details": { "score": 0.945, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/ibm__granite-4-0-micro/helm_safety_ibm_granite_4_0_micro_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "cohere", "id": "cohere/command-r", "name": "Command R", }, "model_route_id": "cohere__command-r", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.943, }, }, "score": 0.943, "score_details": { "score": 0.943, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/cohere__command-r/helm_safety_cohere_command_r_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-small-2501", "name": "Mistral Small 3 2501", }, "model_route_id": "mistralai__mistral-small-2501", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.932, }, }, "score": 0.932, "score_details": { "score": 0.932, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-small-2501/helm_safety_mistralai_mistral_small_2501_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "meta", "id": "meta/llama-3-1-70b-instruct-turbo", "name": "Llama 3.1 Instruct Turbo 70B", }, "model_route_id": "meta__llama-3-1-70b-instruct-turbo", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.925, }, }, "score": 0.925, "score_details": { "score": 0.925, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/meta__llama-3-1-70b-instruct-turbo/helm_safety_meta_llama_3_1_70b_instruct_turbo_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "xai", "id": "xai/grok-4-0709", "name": "Grok 4 0709", }, "model_route_id": "xai__grok-4-0709", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.922, }, }, "score": 0.922, "score_details": { "score": 0.922, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-4-0709/helm_safety_xai_grok_4_0709_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-1106", "name": "GPT-3.5 Turbo 1106", }, "model_route_id": "openai__gpt-3-5-turbo-1106", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.922, }, }, "score": 0.922, "score_details": { "score": 0.922, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-3-5-turbo-1106/helm_safety_openai_gpt_3_5_turbo_1106_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "openai", "id": "openai/gpt-3-5-turbo-0125", "name": "GPT-3.5 Turbo 0125", }, "model_route_id": "openai__gpt-3-5-turbo-0125", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.92, }, }, "score": 0.92, "score_details": { "score": 0.92, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-3-5-turbo-0125/helm_safety_openai_gpt_3_5_turbo_0125_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x22b-instruct-v0-1", "name": "Mixtral Instruct 8x22B", }, "model_route_id": "mistralai__mixtral-8x22b-instruct-v0-1", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.915, }, }, "score": 0.915, "score_details": { "score": 0.915, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mixtral-8x22b-instruct-v0-1/helm_safety_mistralai_mixtral_8x22b_instruct_v0_1_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "mistralai", "id": "mistralai/mixtral-8x7b-instruct-v0-1", "name": "Mixtral Instruct 8x7B", }, "model_route_id": "mistralai__mixtral-8x7b-instruct-v0-1", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.905, }, }, "score": 0.905, "score_details": { "score": 0.905, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mixtral-8x7b-instruct-v0-1/helm_safety_mistralai_mixtral_8x7b_instruct_v0_1_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-instruct-v0-3", "name": "Mistral Instruct v0.3 7B", }, "model_route_id": "mistralai__mistral-7b-instruct-v0-3", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.81, }, }, "score": 0.81, "score_details": { "score": 0.81, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-7b-instruct-v0-3/helm_safety_mistralai_mistral_7b_instruct_v0_3_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "allenai", "id": "allenai/olmo-2-1124-13b-instruct", "name": "OLMo 2 13B Instruct November 2024", }, "model_route_id": "allenai__olmo-2-1124-13b-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.81, }, }, "score": 0.81, "score_details": { "score": 0.81, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/allenai__olmo-2-1124-13b-instruct/helm_safety_allenai_olmo_2_1124_13b_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "allenai", "id": "allenai/olmo-2-1124-7b-instruct", "name": "OLMo 2 7B Instruct November 2024", }, "model_route_id": "allenai__olmo-2-1124-7b-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.775, }, }, "score": 0.775, "score_details": { "score": 0.775, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/allenai__olmo-2-1124-7b-instruct/helm_safety_allenai_olmo_2_1124_7b_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "allenai", "id": "allenai/olmoe-1b-7b-0125-instruct", "name": "OLMoE 1B-7B Instruct January 2025", }, "model_route_id": "allenai__olmoe-1b-7b-0125-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.725, }, }, "score": 0.725, "score_details": { "score": 0.725, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/allenai__olmoe-1b-7b-0125-instruct/helm_safety_allenai_olmoe_1b_7b_0125_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "databricks", "id": "databricks/dbrx-instruct", "name": "DBRX Instruct", }, "model_route_id": "databricks__dbrx-instruct", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.535, }, }, "score": 0.535, "score_details": { "score": 0.535, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/databricks__dbrx-instruct/helm_safety_databricks_dbrx_instruct_1777076383_2276576.json", }, { "evaluation_timestamp": "1777076383.2276576", "model_info": { "developer": "mistralai", "id": "mistralai/mistral-7b-instruct-v0-1", "name": "Mistral Instruct v0.1 7B", }, "model_route_id": "mistralai__mistral-7b-instruct-v0-1", "result": { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "detailed_evaluation_results_url": undefined, "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "evalcards": { "annotations": { "cross_party_divergence": null, "provenance": { "distinct_reporting_organizations": 1, "first_party_only": false, "is_multi_source": false, "signal_version": "1.0", "source_type": "third_party", }, "reproducibility_gap": { "has_reproducibility_gap": true, "missing_fields": [ "temperature", "max_tokens", ], "populated_field_count": 0, "required_field_count": 2, "signal_version": "1.0", }, "variant_divergence": null, }, }, "evaluation_name": "LM Evaluated Safety score", "evaluation_timestamp": "1777076383.2276576", "metric_config": { "evaluation_description": "LM Evaluated Safety score on SimpleSafetyTests", "lower_is_better": false, "max_score": 1, "min_score": 0, "score_type": "continuous", "unit": undefined, }, "metric_key": "lm_evaluated_safety_score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "score_details": { "score": 0.432, }, }, "score": 0.432, "score_details": { "score": 0.432, }, "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_metadata": { "evaluator_relationship": "third_party", "source_name": "helm_safety", "source_organization_name": "crfm", "source_type": "documentation", }, "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/mistralai__mistral-7b-instruct-v0-1/helm_safety_mistralai_mistral_7b_instruct_v0_1_1777076383_2276576.json", }, ], "models_count": 87, "provenance_summary": { "first_party_only_groups": 0, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 0, "third_party": 87, "unspecified": 0, }, "total_groups": 85, "total_results": 87, }, "reproducibility_summary": { "has_reproducibility_gap_count": 87, "populated_ratio_avg": 0, "results_total": 87, }, "root_metrics": [ { "canonical_display_name": "SimpleSafetyTests / LM Evaluated Safety score", "display_name": "SimpleSafetyTests / LM Evaluated Safety score", "lower_is_better": false, "metric_key": "lm_evaluated_safety_score", "metric_name": "LM Evaluated Safety score", "metric_summary_id": "helm_safety_simplesafetytests_lm_evaluated_safety_score", "models_count": 87, "top_score": 1, "unit": undefined, }, ], "source_data": { "dataset_name": "SimpleSafetyTests", "source_type": "url", "url": [ "https://storage.googleapis.com/crfm-helm-public/safety/benchmark_output/releases/v1.17.0/groups/safety_scenarios.json", ], }, "source_types": [], "subtasks": [], "third_party_ratio": 0, "worst_model": { "name": "Mistral Instruct v0.1 7B", "score": 0.432, }, } `; exports[`hfModelCardToEvaluationCardData > 01-ai__yi-34b — Developer name canonicalization (01-ai → 01.AI per KNOWN_DEVELOPER_NAMES) 1`] = ` { "benchmark_names": [ "BBH", "GPQA", "GSM8K", "HELM Lite", "IFEval", "LegalBench", "MATH", "MATH Level 5", "MMLU", "MMLU PRO", "MUSR", "MedQA", "NarrativeQA", "NaturalQuestions (closed Book)", "OpenbookQA", "WMT 2014", ], "benchmarks_count": 3, "canonical_model_name": "Yi 34B", "categories": [ "General", "Knowledge", "Reasoning", ], "category_stats": { "General": 1, "Knowledge": 1, "Reasoning": 1, }, "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 0, "total_groups": 52, "variant_divergent_count": 0, }, "detail_urls": [], "developer": "01-ai", "eval_libraries": [], "evaluations_count": 3, "evaluator_count": 0, "evaluator_names": [], "evidence_count": 3, "id": "01-ai/yi-34b", "independent_verification_ratio": 0, "latest_source_name": "16 benchmarks", "latest_timestamp": "2026-03-21T12:31:52.005480Z", "missing_generation_config_count": 0, "model_id": "01-ai/yi-34b", "model_name": "Yi 34B", "params_billions": 34, "provenance_summary": { "first_party_only_groups": 0, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 0, "third_party": 52, "unspecified": 0, }, "total_groups": 52, "total_results": 52, }, "reproducibility_status": "partial", "reproducibility_summary": { "has_reproducibility_gap_count": 52, "populated_ratio_avg": 0, "results_total": 52, }, "route_id": "01-ai%2Fyi-34b", "score_summary": { "average": 0.6793153846153845, "count": 52, "max": 0.936, "min": 0.0514, }, "source_type_count": 1, "source_types": [ "documentation", ], "source_urls": [], "third_party_eval_count": 0, "top_scores": [ { "benchmark": "MMLU", "benchmarkKey": "helm_mmlu", "metric": "Marketing", "score": 0.936, }, { "benchmark": "OpenbookQA", "benchmarkKey": "helm_lite_openbookqa", "metric": "OpenbookQA", "score": 0.92, }, { "benchmark": "NarrativeQA", "benchmarkKey": "helm_lite_narrativeqa", "metric": "NarrativeQA", "score": 0.782, }, { "benchmark": "MedQA", "benchmarkKey": "helm_lite_medqa", "metric": "MedQA", "score": 0.656, }, { "benchmark": "MMLU", "benchmarkKey": "helm_lite_mmlu", "metric": "MMLU", "score": 0.65, }, { "benchmark": "GSM8K", "benchmarkKey": "helm_lite_gsm8k", "metric": "GSM8K", "score": 0.648, }, { "benchmark": "LegalBench", "benchmarkKey": "helm_lite_legalbench", "metric": "LegalBench", "score": 0.618, }, { "benchmark": "HELM Lite", "benchmarkKey": "helm_lite", "metric": "helm_lite", "score": 0.57, }, { "benchmark": "BBH", "benchmarkKey": "hfopenllm_v2_bbh", "metric": "BBH", "score": 0.5457, }, { "benchmark": "NaturalQuestions (closed Book)", "benchmarkKey": "helm_lite_naturalquestions_closed_book", "metric": "NaturalQuestions (closed-book)", "score": 0.443, }, { "benchmark": "MMLU PRO", "benchmarkKey": "hfopenllm_v2_mmlu_pro", "metric": "MMLU-PRO", "score": 0.4412, }, { "benchmark": "MUSR", "benchmarkKey": "hfopenllm_v2_musr", "metric": "MUSR", "score": 0.4119, }, { "benchmark": "MATH", "benchmarkKey": "helm_lite_math", "metric": "MATH", "score": 0.375, }, { "benchmark": "GPQA", "benchmarkKey": "hfopenllm_v2_gpqa", "metric": "GPQA", "score": 0.3666, }, { "benchmark": "IFEval", "benchmarkKey": "hfopenllm_v2_ifeval", "metric": "IFEval", "score": 0.3046, }, ], "variant_count": 1, } `; exports[`hfModelCardToEvaluationCardData > anthropic__claude-opus-4.5 — Dotted route_id (vs dashed in model detail files) — capturing the route-id mismatch 1`] = ` { "benchmark_names": [ "ARC Prize evaluations leaderboard JSON", "ARC AGI v2", "AppWorld Benchmark", "Artificial Analysis LLM API", "BFCL leaderboard CSV", "BrowseComp Plus", "MCP Atlas", "MMMLU", "MMMU (validation)", "OSWorld", "SWE Bench Verified", "SWE Bench Verified", "SWE-bench", "Tau2 Retail", "Tau2 Telecom", "Terminal Bench 2.0", "Terminal Bench 2.0", "Τ Bench (Tool Agent User Interaction Benchmark)", ], "benchmarks_count": 10, "canonical_model_name": "Claude Opus 4.5", "categories": [ "Agentic", "General", "Knowledge", "Reasoning", ], "category_stats": { "Agentic": 12, "General": 12, "Knowledge": 12, "Reasoning": 13, }, "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 6, "total_groups": 76, "variant_divergent_count": 6, }, "detail_urls": [], "developer": "Anthropic", "eval_libraries": [], "evaluations_count": 49, "evaluator_count": 0, "evaluator_names": [], "evidence_count": 49, "id": "anthropic/claude-opus-4.5", "independent_verification_ratio": 0, "latest_source_name": "18 benchmarks", "latest_timestamp": "2026-04-25T09:07:44.422824Z", "missing_generation_config_count": 0, "model_id": "anthropic/claude-opus-4.5", "model_name": "Claude Opus 4.5", "params_billions": null, "provenance_summary": { "first_party_only_groups": 9, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 9, "third_party": 155, "unspecified": 0, }, "total_groups": 76, "total_results": 164, }, "reproducibility_status": "partial", "reproducibility_summary": { "has_reproducibility_gap_count": 164, "populated_ratio_avg": 0, "results_total": 164, }, "route_id": "anthropic%2Fclaude-opus-4.5", "score_summary": { "average": 25.281729268292686, "count": 164, "max": 95.5, "min": 0.0708, }, "source_type_count": 1, "source_types": [ "documentation", ], "source_urls": [], "third_party_eval_count": 0, "top_scores": [ { "benchmark": "Terminal Bench 2.0", "benchmarkKey": "terminal_bench_2_0", "metric": "terminal-bench-2.0", "score": 63.1, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "metric": "artificial_analysis.artificial_analysis_math_index", "score": 62.7, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "metric": "artificial_analysis.median_output_tokens_per_second", "score": 52.885, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "metric": "artificial_analysis.artificial_analysis_intelligence_index", "score": 43.1, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "metric": "artificial_analysis.artificial_analysis_coding_index", "score": 42.9, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "metric": "artificial_analysis.price_1m_output_tokens", "score": 25, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "metric": "artificial_analysis.price_1m_blended_3_to_1", "score": 10, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "metric": "artificial_analysis.price_1m_input_tokens", "score": 5, }, { "benchmark": "BFCL leaderboard CSV", "benchmarkKey": "bfcl", "metric": "format_sensitivity", "score": 3.65, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "metric": "artificial_analysis.median_time_to_first_token_seconds", "score": 1.311, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "metric": "artificial_analysis.median_time_to_first_answer_token", "score": 1.311, }, { "benchmark": "Tau2 Telecom", "benchmarkKey": "llm_stats_tau2_telecom", "metric": "llm_stats.tau2-telecom", "score": 0.982, }, { "benchmark": "MMMLU", "benchmarkKey": "llm_stats_mmmlu", "metric": "llm_stats.mmmlu", "score": 0.908, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_mmlu_pro", "metric": "artificial_analysis.mmlu_pro", "score": 0.889, }, { "benchmark": "Tau2 Retail", "benchmarkKey": "llm_stats_tau2_retail", "metric": "llm_stats.tau2-retail", "score": 0.889, }, ], "variant_count": 3, } `; exports[`hfModelCardToEvaluationCardData > openai__gpt-5 — 6 variants; rich top_benchmark_scores; hfModelCardToEvaluationCardData edge case 1`] = ` { "benchmark_names": [ "ACE", "APEX Agents", "APEX v1", "ARC Prize evaluations leaderboard JSON", "Aider Polyglot", "Anthropic Red Team", "Artificial Analysis LLM API", "BBQ", "BrowseComp Long Context 128k", "BrowseComp Long Context 256k", "COLLIE", "CharXiv R", "ERQA", "Easy Problems", "FActScore", "Fibble Arena", "FrontierMath", "GPQA", "Global MMLU Lite", "Graphwalks BFS <128k", "Graphwalks parents <128k", "HMMT 2025", "Hard Problems", "HarmBench", "HealthBench Hard", "Helm air bench", "Helm safety", "Holistic Evaluation of Language Models (HELM)", "HumanEval", "IFEval", "Internal API instruction following (hard)", "LongFact Concepts", "LongFact Objects", "MATH", "MMLU", "MMLU Pro", "MMMU", "MMMU Pro", "Medium Problems", "Multi Challenge", "Multi SWE Bench (c++)", "Omni MATH", "OpenAI MRCR: 2 needle 128k", "OpenAI MRCR: 2 needle 256k", "SWE Lancer (IC Diamond subset)", "SWE PolyBench Verified (Java)", "SWE PolyBench Verified (JavaScript)", "SWE PolyBench Verified (Python)", "SWE PolyBench Verified (TypeScript)", "SWE Bench Verified", "SciArena leaderboard API", "SimpleSafetyTests", "Tau2 Airline", "Tau2 Retail", "Tau2 Telecom", "Terminal Bench 2.0", "VideoMME w sub.", "VideoMMMU", "WildBench", "Wordle Arena", "XSTest", ], "benchmarks_count": 18, "canonical_model_name": "GPT 5", "categories": [ "Agentic", "General", "Knowledge", "Reasoning", ], "category_stats": { "Agentic": 7, "General": 7, "Knowledge": 7, "Reasoning": 9, }, "comparability_summary": { "cross_party_divergent_count": 0, "groups_with_cross_party_check": 0, "groups_with_variant_check": 0, "total_groups": 508, "variant_divergent_count": 0, }, "detail_urls": [], "developer": "OpenAI", "eval_libraries": [], "evaluations_count": 30, "evaluator_count": 0, "evaluator_names": [], "evidence_count": 30, "id": "openai/gpt-5", "independent_verification_ratio": 0, "latest_source_name": "61 benchmarks", "latest_timestamp": "2026-04-25T09:07:44.422824Z", "missing_generation_config_count": 0, "model_id": "openai/gpt-5", "model_name": "GPT 5", "params_billions": null, "provenance_summary": { "first_party_only_groups": 43, "multi_source_groups": 0, "source_type_distribution": { "collaborative": 0, "first_party": 44, "third_party": 515, "unspecified": 0, }, "total_groups": 508, "total_results": 559, }, "reproducibility_status": "partial", "reproducibility_summary": { "has_reproducibility_gap_count": 541, "populated_ratio_avg": 0.03220035778175313, "results_total": 559, }, "route_id": "openai%2Fgpt-5", "score_summary": { "average": 673.8712930083715, "count": 559, "max": 73320, "min": 0, }, "source_type_count": 1, "source_types": [ "documentation", ], "source_urls": [], "third_party_eval_count": 0, "top_scores": [ { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_output_tokens_per_second", "metric": "artificial_analysis.median_output_tokens_per_second", "score": 95.722, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_math_index", "metric": "artificial_analysis.artificial_analysis_math_index", "score": 94.3, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_token_seconds", "metric": "artificial_analysis.median_time_to_first_token_seconds", "score": 82.082, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_median_time_to_first_answer_token", "metric": "artificial_analysis.median_time_to_first_answer_token", "score": 82.082, }, { "benchmark": "Terminal Bench 2.0", "benchmarkKey": "terminal_bench_2_0", "metric": "terminal-bench-2.0", "score": 49.6, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_intelligence_index", "metric": "artificial_analysis.artificial_analysis_intelligence_index", "score": 44.6, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_artificial_analysis_coding_index", "metric": "artificial_analysis.artificial_analysis_coding_index", "score": 36, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_output_tokens", "metric": "artificial_analysis.price_1m_output_tokens", "score": 10, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_blended_3_to_1", "metric": "artificial_analysis.price_1m_blended_3_to_1", "score": 3.438, }, { "benchmark": "SciArena leaderboard API", "benchmarkKey": "sciarena", "metric": "overall_cost_per_100_calls_usd", "score": 2.9752, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_price_1m_input_tokens", "metric": "artificial_analysis.price_1m_input_tokens", "score": 1.25, }, { "benchmark": "Helm air bench", "benchmarkKey": "helm_air_bench", "metric": "AIRBench 2024 - Self-harm", "score": 1, }, { "benchmark": "SWE Lancer (IC Diamond subset)", "benchmarkKey": "llm_stats_swe_lancer_ic_diamond_subset", "metric": "llm_stats.swe-lancer-ic-diamond-subset", "score": 1, }, { "benchmark": "SimpleSafetyTests", "benchmarkKey": "helm_safety_simplesafetytests", "metric": "SimpleSafetyTests", "score": 0.998, }, { "benchmark": "Artificial Analysis LLM API", "benchmarkKey": "artificial_analysis_llms_artificial_analysis_math_500", "metric": "artificial_analysis.math_500", "score": 0.994, }, ], "variant_count": 6, } `;