{
  "benchmark_card": {
    "benchmark_details": {
      "name": "MMLU-Pro",
      "overview": "MMLU-Pro is an enhanced version of the Massive Multitask Language Understanding benchmark designed to be more challenging and robust. It measures multi-task language understanding and reasoning capabilities by integrating more reasoning-focused questions, expanding answer choices from four to ten, and eliminating trivial or noisy questions found in its predecessor. It covers a broad range of subjects.",
      "data_type": "tabular, text",
      "domains": [
        "STEM",
        "humanities",
        "social sciences",
        "math",
        "physics",
        "chemistry",
        "law",
        "engineering",
        "economics",
        "health",
        "psychology",
        "business",
        "biology",
        "philosophy",
        "computer science",
        "history"
      ],
      "languages": [
        "English"
      ],
      "similar_benchmarks": [
        "MMLU"
      ],
      "resources": [
        "https://arxiv.org/abs/2406.01574",
        "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro",
        "https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro",
        "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
      ]
    },
    "purpose_and_intended_users": {
      "goal": "To provide a more challenging and discriminative benchmark for tracking progress in language model capabilities, testing deeper cognitive processes and reasoning as models plateau on existing benchmarks.",
      "audience": [
        "Researchers evaluating large language models"
      ],
      "tasks": [
        "Multiple-choice question answering across a broad range of subjects"
      ],
      "limitations": "The dataset contains some mistakes and formatting inconsistencies, which the maintainers are correcting based on expert feedback.",
      "out_of_scope_uses": [
        "Not specified"
      ]
    },
    "data": {
      "source": "The dataset is an enhanced version of MMLU, integrating more challenging questions from multiple sources. These include original MMLU questions (with trivial or ambiguous ones removed), hand-picked high-quality STEM problems from the internet, human-annotated questions requiring theorems from TheoremQA, and science questions from college exams via SciBench.",
      "size": "The test split contains 12,032 examples, placing the dataset in the 10K to 100K size category. The total file size is 8,775,905 bytes.",
      "format": "The data is stored in Parquet format and consists of multiple-choice questions, expanding the number of answer choices from four (as in MMLU) to ten.",
      "annotation": "The annotation process involved expert review. Over ten experts scrutinized each question and its options to ensure challenge, comprehensiveness, accuracy, and fairness. Some answers were later corrected based on recommendations from specialists such as medical professionals."
    },
    "methodology": {
      "methods": [
        "Models are evaluated using multiple-choice question answering, typically with a 5-shot prompting setup, though some models use 0-shot.",
        "Performance is tested using both direct answering and Chain-of-Thought (CoT) reasoning.",
        "The benchmark tests performance under 24 different prompt styles."
      ],
      "metrics": [
        "Accuracy (reported as a percentage or score)"
      ],
      "calculation": "The overall score is an average accuracy across subjects, but the specific calculation method is not detailed.",
      "interpretation": "Higher scores indicate better performance. Performance on MMLU-Pro is significantly lower than on MMLU, indicating it is more challenging.",
      "baseline_results": "PAPER baselines: GPT-4o achieves over 70% accuracy in Math and Physics subjects. Llama-3-70B-Instruct achieves an overall accuracy of 56.2%. Gemma-7B and Mistral-7B-v0.1 have lower performance (e.g., Mistral-7B-v0.1 scores just over 20% in Math and Physics). HF_README baselines: GPT-4o achieves an overall score of 0.7255 with CoT and 0.5346 with direct prompting. Other reported scores include Claude-3-Opus (0.6845), Claude-3-Sonnet (0.5511), Gemini 1.5 Flash (0.5912), and Llama-3-70B-Instruct (0.5620). EEE results: OLMo 2 32B Instruct March 2025 scored 0.4140.",
      "validation": "The benchmark demonstrates greater stability under varying prompts, with sensitivity to prompt variations decreasing from 4-5% in MMLU to just 2% in MMLU-Pro."
    },
    "ethical_and_legal_considerations": {
      "privacy_and_anonymity": "Not specified",
      "data_licensing": "MIT License",
      "consent_procedures": "Not specified",
      "compliance_with_regulations": "Not specified"
    },
    "possible_risks": [
      {
        "category": "Over- or under-reliance",
        "description": [
          "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
      },
      {
        "category": "Unrepresentative data",
        "description": [
          "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
      },
      {
        "category": "Uncertain data provenance",
        "description": [
          "Data provenance refers to the traceability of data (including synthetic data), which includes its ownership, origin, transformations, and generation. Proving that the data is the same as the original source with correct usage terms is difficult without standardized methods for verifying data sources or generation."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-provenance.html"
      },
      {
        "category": "Data bias",
        "description": [
          "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
      },
      {
        "category": "Improper usage",
        "description": [
          "Improper usage occurs when a model is used for a purpose that it was not originally designed for."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html"
      }
    ],
    "flagged_fields": {},
    "missing_fields": [
      "purpose_and_intended_users.out_of_scope_uses",
      "ethical_and_legal_considerations.privacy_and_anonymity",
      "ethical_and_legal_considerations.consent_procedures",
      "ethical_and_legal_considerations.compliance_with_regulations"
    ],
    "card_info": {
      "created_at": "2026-03-17T16:20:31.763989",
      "llm": "deepseek-ai/DeepSeek-V3.2"
    }
  }
}