Spaces:
Running
Running
File size: 6,331 Bytes
04b4cff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | {
"benchmark_card": {
"benchmark_details": {
"name": "Measuring Massive Multitask Language Understanding (MMLU)",
"overview": "MMLU is a multiple-choice question-answering benchmark that measures a text model's multitask accuracy across 57 distinct tasks. It is designed to test a wide range of knowledge and problem-solving abilities, covering diverse academic and professional subjects from elementary to advanced levels.",
"data_type": "text",
"domains": [
"STEM",
"humanities",
"social sciences"
],
"languages": [
"English"
],
"similar_benchmarks": [
"GLUE",
"SuperGLUE"
],
"resources": [
"https://arxiv.org/abs/2009.03300",
"https://huggingface.co/datasets/cais/mmlu",
"https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json",
"https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
]
},
"purpose_and_intended_users": {
"goal": "To bridge the gap between the wide-ranging knowledge models acquire during pretraining and existing evaluation measures by assessing models across a diverse set of academic and professional subjects.",
"audience": [
"Researchers analyzing model capabilities and identifying shortcomings"
],
"tasks": [
"Multiple-choice question answering"
],
"limitations": "Models exhibit lopsided performance, frequently do not know when they are wrong, and have near-random accuracy on some socially important subjects like morality and law.",
"out_of_scope_uses": [
"Not specified"
]
},
"data": {
"source": "The dataset is an original source with expert-generated questions.",
"size": "The dataset contains over 100,000 examples, with a test split of 14,042 examples, a validation split of 1,531 examples, a dev split of 285 examples, and an auxiliary training split of 99,842 examples.",
"format": "parquet",
"annotation": "The dataset has no additional annotations; each question provides the correct answer as a class label (A, B, C, or D)."
},
"methodology": {
"methods": [
"Models are evaluated exclusively in zero-shot and few-shot settings to measure knowledge acquired during pretraining."
],
"metrics": [
"MMLU (accuracy)"
],
"calculation": "The overall score is an average accuracy across the 57 tasks.",
"interpretation": "Higher scores indicate better performance. Near random-chance accuracy indicates weak performance. The very largest GPT-3 model improved over random chance by almost 20 percentage points on average, but models still need substantial improvements to reach expert-level accuracy.",
"baseline_results": "Paper baselines: Most recent models have near random-chance accuracy. The very largest GPT-3 model improved over random chance by almost 20 percentage points on average. EEE results: Yi 34B scored 0.6500, Anthropic-LM v4-s3 52B scored 0.4810. The mean score across 2 evaluated models is 0.5655.",
"validation": "Not specified"
},
"ethical_and_legal_considerations": {
"privacy_and_anonymity": "Not specified",
"data_licensing": "MIT License",
"consent_procedures": "Not specified",
"compliance_with_regulations": "Not specified"
},
"possible_risks": [
{
"category": "Over- or under-reliance",
"description": [
"In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
],
"url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
},
{
"category": "Unrepresentative data",
"description": [
"Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
],
"url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
},
{
"category": "Data bias",
"description": [
"Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
],
"url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
},
{
"category": "Lack of data transparency",
"description": [
"Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation.\u00a0"
],
"url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
},
{
"category": "Improper usage",
"description": [
"Improper usage occurs when a model is used for a purpose that it was not originally designed for."
],
"url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html"
}
],
"flagged_fields": {},
"missing_fields": [
"purpose_and_intended_users.out_of_scope_uses",
"methodology.validation",
"ethical_and_legal_considerations.privacy_and_anonymity",
"ethical_and_legal_considerations.consent_procedures",
"ethical_and_legal_considerations.compliance_with_regulations"
],
"card_info": {
"created_at": "2026-03-17T13:14:49.605975",
"llm": "deepseek-ai/DeepSeek-V3.2"
}
}
} |