{
  "benchmark_card": {
    "benchmark_details": {
      "name": "MuSR",
      "overview": "MuSR (Multistep Soft Reasoning) is a benchmark that measures the ability of language models to perform multistep soft reasoning based on natural language narratives. It combines sophisticated narratives with complex reasoning that requires commonsense knowledge and is not solvable by simple rule-based systems. The benchmark consists of three tasks: murder mysteries, object placements, and team allocations.",
      "data_type": "text",
      "domains": [
        "reasoning",
        "commonsense reasoning",
        "planning"
      ],
      "languages": [
        "English"
      ],
      "similar_benchmarks": [
        "bAbI",
        "BigTOM",
        "ToMi",
        "RuleTakers",
        "ProntoQA",
        "SocialIQA",
        "StrategyQA"
      ],
      "resources": [
        "https://github.com/Zayne-Sprague/MuSR",
        "https://arxiv.org/abs/2310.16049",
        "https://huggingface.co/datasets/TAUR-Lab/MuSR"
      ]
    },
    "purpose_and_intended_users": {
      "goal": "To evaluate and characterize the gaps in language models' abilities to perform robust, multistep reasoning in complex natural language settings, particularly testing the limits of techniques like chain-of-thought.",
      "audience": [
        "Researchers evaluating language models",
        "Researchers evaluating neurosymbolic systems on reasoning capabilities"
      ],
      "tasks": [
        "Question answering",
        "Solving murder mysteries",
        "Solving object placement problems",
        "Solving team allocation problems"
      ],
      "limitations": "The benchmark instances are generated by GPT-4, which may lead to simple, poor-quality narratives with potential inconsistencies, though the paper argues they are valid test cases if the underlying information is faithfully preserved.",
      "out_of_scope_uses": [
        "Not specified"
      ]
    },
    "data": {
      "source": "The data is synthetically generated using a neurosymbolic synthetic-to-natural generation algorithm that employs GPT-4. The process begins with gold facts, constructs a reasoning tree, and iteratively generates a narrative.",
      "size": "756 instances across three domains: 250 for Murder Mystery, 256 for Object Placements, and 250 for Team Allocations. The dataset is categorized as containing fewer than 1,000 examples (n<1K).",
      "format": "CSV, containing free-text narratives (ranging from hundreds to roughly 1000 words in length) followed by a multiple-choice question.",
      "annotation": "Answers are derived from the underlying gold facts used in generation. For validation, human annotators (7 total) solved instances using a chain-of-thought+ prompt, with instances triply-annotated (34-40 per domain). Human accuracy was very high, with the lowest average annotator score at 90% and majority vote accuracy between 94.1% and 100%."
    },
    "methodology": {
      "methods": [
        "Models are evaluated in zero-shot and single-shot (1-shot) settings.",
        "Prompting strategies include single-shot prompting, chain-of-thought (CoT), and an engineered variant called CoT+.",
        "Neurosymbolic algorithms like Program-Aided Language Models (PAL) and SymbolicTOM are also evaluated on compatible domains."
      ],
      "metrics": [
        "MUSR (Accuracy)"
      ],
      "calculation": "Performance is measured as accuracy (percentage correct) for each of the three domains (MM, OP, TA) separately. The paper does not report a single aggregated score.",
      "interpretation": "Higher accuracy indicates better performance. Human performance sets a high ceiling (ranging from 94.1% to 100% by majority vote), while random baselines are at or near chance (ranging from 24.6% to 50%).",
      "baseline_results": "Paper baselines: Random baseline (MM: 50%, OP: 24.6%, TA: 33.3%), GPT-4 (80.4%, 60.9%, 68.4%), GPT-3.5 (61.6%, 46.9%, 40.4%), Llama2 70b Chat (48.8%, 42.2%, 44.8%), Llama2 7b Chat (50.8%, 29.3%, 36.8%), Vicuna 7b v1.5 (48.4%, 29.7%, 26.4%), Vicuna 13b v1.5 (50.8%, 34.4%, 32%), Vicuna 33b v1.3 (49.6%, 31.2%, 30%), Human Eval (94.1%, 95%, 100%). PAL results for Team Allocation outperform end-to-end models but are below human performance. EEE results: YiSM-blossom5.1-34B-SLERP achieved an accuracy of 0.4413.",
      "validation": "The dataset is validated by measuring human annotator performance to ensure the narratives support the intended reasoning. Rule-based baselines (e.g., picking the suspect with the longest chapter) are used as sanity checks and perform near random chance, confirming the tasks are not trivially solvable."
    },
    "ethical_and_legal_considerations": {
      "privacy_and_anonymity": "Not specified",
      "data_licensing": "Creative Commons Attribution 4.0",
      "consent_procedures": "Annotation was performed by three hired undergraduate students. No details on compensation or the hiring platform are provided.",
      "compliance_with_regulations": "The research received support from NSF, DARPA, and the Air Force Research Laboratory. No mention of IRB approval or other specific regulatory compliance is made."
    },
    "possible_risks": [
      {
        "category": "Over- or under-reliance",
        "description": [
          "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
      },
      {
        "category": "Unrepresentative data",
        "description": [
          "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
      },
      {
        "category": "Data contamination",
        "description": [
          "Data contamination occurs when incorrect data is used for training. For example, data that is not aligned with model's purpose or data that is already set aside for other development tasks such as testing and evaluation."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-contamination.html"
      },
      {
        "category": "Reproducibility",
        "description": [
          "Replicating agent behavior or output can be impacted by changes or updates made to external services and tools. This impact is increased if the agent is built with generative AI."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/reproducibility-agentic.html"
      },
      {
        "category": "Improper usage",
        "description": [
          "Improper usage occurs when a model is used for a purpose that it was not originally designed for."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html"
      }
    ],
    "flagged_fields": {
      "methodology.calculation": "[Possible Hallucination], no supporting evidence found in source material",
      "methodology.baseline_results": "[Possible Hallucination], no supporting evidence found in source material",
      "methodology.validation": "[Factuality Score: 0.17], low factual alignment with source material"
    },
    "missing_fields": [
      "purpose_and_intended_users.out_of_scope_uses",
      "ethical_and_legal_considerations.privacy_and_anonymity"
    ],
    "card_info": {
      "created_at": "2026-03-17T12:25:43.235734",
      "llm": "deepseek-ai/DeepSeek-V3.2"
    }
  }
}