{
  "benchmark_card": {
    "benchmark_details": {
      "name": "MEDQA",
      "overview": "MEDQA is a free-form multiple-choice open-domain question answering (OpenQA) benchmark designed to measure a model's ability to solve medical problems. It is distinctive as the first such dataset sourced from professional medical board exams, covering multiple languages and presenting a challenging real-world scenario.",
      "data_type": "text",
      "domains": [
        "medical knowledge",
        "professional medical exams"
      ],
      "languages": [
        "English"
      ],
      "similar_benchmarks": [
        "ARC",
        "OpenBookQA"
      ],
      "resources": [
        "https://github.com/jind11/MedQA",
        "https://arxiv.org/abs/2009.13081",
        "https://huggingface.co/datasets/GBaker/MedQA-USMLE-4-options",
        "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
      ]
    },
    "purpose_and_intended_users": {
      "goal": "To present a challenging open-domain question answering dataset to promote the development of stronger models capable of handling sophisticated real-world medical scenarios, specifically evaluating performance on medical knowledge as tested in professional exams.",
      "audience": [
        "The natural language processing (NLP) community"
      ],
      "tasks": [
        "Free-form multiple-choice question answering",
        "Open-domain question answering"
      ],
      "limitations": "Even the best current methods achieve relatively low accuracy (36.7% to 70.1% across languages), indicating the benchmark's difficulty and the limitations of existing models.",
      "out_of_scope_uses": [
        "Not specified"
      ]
    },
    "data": {
      "source": "The data is collected from professional medical board exams.",
      "size": "The dataset contains 12,723 questions in English, 34,251 in simplified Chinese, and 14,123 in traditional Chinese. The total number of examples falls within the 10K to 100K range.",
      "format": "JSON",
      "annotation": "The answer labels are the correct answers from the professional exams. No additional annotation process is described."
    },
    "methodology": {
      "methods": [
        "The benchmark uses a sequential combination of a document retriever and a machine comprehension model. It includes both rule-based and neural methods.",
        "The evaluation is a standard question-answering task, though the specific learning setting (e.g., zero-shot, few-shot, fine-tuning) is not explicitly defined."
      ],
      "metrics": [
        "Accuracy"
      ],
      "calculation": "The overall score is the accuracy on the test set.",
      "interpretation": "Higher accuracy indicates better performance. The best reported accuracies are 36.7% for English, 42.0% for traditional Chinese, and 70.1% for simplified Chinese questions.",
      "baseline_results": "Original paper baselines: The best method reported achieves 36.7% accuracy on English, 42.0% on traditional Chinese, and 70.1% on simplified Chinese questions. Model names are not specified. EEE results: Yi 34B achieves a score of 0.656 (65.6%).",
      "validation": "Not specified"
    },
    "ethical_and_legal_considerations": {
      "privacy_and_anonymity": "Not specified",
      "data_licensing": "Creative Commons Attribution 4.0",
      "consent_procedures": "Not specified",
      "compliance_with_regulations": "Not specified"
    },
    "possible_risks": [
      {
        "category": "Over- or under-reliance",
        "description": [
          "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
      },
      {
        "category": "Unrepresentative data",
        "description": [
          "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
      },
      {
        "category": "Uncertain data provenance",
        "description": [
          "Data provenance refers to the traceability of data (including synthetic data), which includes its ownership, origin, transformations, and generation. Proving that the data is the same as the original source with correct usage terms is difficult without standardized methods for verifying data sources or generation."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-provenance.html"
      },
      {
        "category": "Data bias",
        "description": [
          "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
      },
      {
        "category": "Lack of data transparency",
        "description": [
          "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation.\u00a0"
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
      }
    ],
    "flagged_fields": {},
    "missing_fields": [
      "purpose_and_intended_users.out_of_scope_uses",
      "methodology.validation",
      "ethical_and_legal_considerations.privacy_and_anonymity",
      "ethical_and_legal_considerations.consent_procedures",
      "ethical_and_legal_considerations.compliance_with_regulations"
    ],
    "card_info": {
      "created_at": "2026-03-17T13:23:29.822123",
      "llm": "deepseek-ai/DeepSeek-V3.2"
    }
  }
}