metadata/benchmark_card_WildBench.json

{
  "benchmark_card": {
    "benchmark_details": {
      "name": "WildBench",
      "overview": "WildBench is a benchmark that measures the performance of large language models on challenging, open-ended tasks derived from real-world user queries. It consists of 1,024 tasks selected from human-chatbot conversation logs. Its distinctive features include using a natural distribution of real user queries and employing automated evaluation with two novel metrics, WB-Reward and WB-Score, which utilize task-specific checklists and structured explanations.",
      "data_type": "tabular, text",
      "domains": [
        "Info Seeking",
        "Math & Data",
        "Reasoning & Planning",
        "Creative Tasks"
      ],
      "languages": [
        "English"
      ],
      "similar_benchmarks": [
        "AlpacaEval",
        "ArenaHard",
        "MT-bench",
        "Chatbot Arena"
      ],
      "resources": [
        "https://arxiv.org/abs/2406.04770",
        "https://huggingface.co/datasets/allenai/WildBench",
        "https://huggingface.co/spaces/allenai/WildBench",
        "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
      ]
    },
    "purpose_and_intended_users": {
      "goal": "To provide a realistic, automated, and cost-effective evaluation framework for large language models that reflects their capabilities on challenging, real-world user tasks. It aims to be contamination-resilient and dynamically updated.",
      "audience": [
        "Researchers and practitioners evaluating large language models"
      ],
      "tasks": [
        "Open-ended text generation in response to diverse user queries"
      ],
      "limitations": "The benchmark acknowledges a common issue in LLM-as-a-judge evaluations: bias towards longer outputs. It introduces a length-penalty method to mitigate this.",
      "out_of_scope_uses": [
        "Not specified"
      ]
    },
    "data": {
      "source": "The data is derived from real user-chatbot conversation logs collected by the AI2 WildChat project. Over one million logs were filtered to create the benchmark.",
      "size": "The benchmark contains 1,024 tasks. A separate 'v2-hard' configuration contains 256 examples. The dataset falls into the size category of 1K to 10K examples.",
      "format": "The data is stored in Parquet format.",
      "annotation": "Human annotation was used for quality control. GPT-4-Turbo assisted by summarizing query intents to help human reviewers filter out nonsensical tasks. Human reviewers manually assessed tasks to ensure they were challenging, diverse, and that associated checklist questions were clear. The final set of tasks was retained after this process. Each example includes fine-grained annotations such as task types and checklists for evaluating response quality."
    },
    "methodology": {
      "methods": [
        "Models are evaluated automatically using LLM-as-a-judge, with GPT-4-turbo as the evaluator. The evaluation is not fine-tuning-based and appears to be zero-shot or prompted.",
        "The evaluation uses length-penalized pairwise comparisons and individual scoring to prevent bias towards longer outputs."
      ],
      "metrics": [
        "WB-Reward (for pairwise comparisons)",
        "WB-Score (for individual scoring)",
        "WB-Elo (for leaderboard ranking, merging WB-Reward and WB-Score)"
      ],
      "calculation": "WB-Reward compares a test model against three baseline models of varying performance, producing outcomes of 'much better', 'slightly better', 'tie', 'slightly worse', or 'much worse'. A length penalty converts 'slightly better/worse' to 'tie' if the winner's response is excessively longer. WB-Score evaluates individual model outputs. WB-Elo merges the pairwise comparisons from WB-Reward and WB-Score to perform Elo rating updates.",
      "interpretation": "Higher scores on WB-Reward and WB-Score indicate better performance. Strong performance is validated by a high correlation with human-voted Elo ratings from Chatbot Arena, with WB-Reward achieving a Pearson correlation of 0.98 and WB-Score reaching 0.95.",
      "baseline_results": "Paper baselines: The paper reports results for 40 LLMs but does not list specific scores. It notes that using Haiku as a baseline yields the best correlation with human ratings. EEE results: OLMo 2 32B Instruct March 2025 scored 0.7340 on WildBench.",
      "validation": "The evaluation metrics are validated by their strong correlation with human-voted Elo ratings from Chatbot Arena on hard tasks. Checklist questions used in the evaluation were manually verified for clarity and relevance."
    },
    "ethical_and_legal_considerations": {
      "privacy_and_anonymity": "Not specified",
      "data_licensing": "The dataset is made available under a CC BY license, intended for research and educational use in accordance with AI2's Responsible Use Guidelines.",
      "consent_procedures": "Not specified",
      "compliance_with_regulations": "Not specified"
    },
    "possible_risks": [
      {
        "category": "Over- or under-reliance",
        "description": [
          "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
      },
      {
        "category": "Unrepresentative data",
        "description": [
          "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
      },
      {
        "category": "Data bias",
        "description": [
          "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
      },
      {
        "category": "Data contamination",
        "description": [
          "Data contamination occurs when incorrect data is used for training. For example, data that is not aligned with model's purpose or data that is already set aside for other development tasks such as testing and evaluation."
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-contamination.html"
      },
      {
        "category": "Lack of data transparency",
        "description": [
          "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation.\u00a0"
        ],
        "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
      }
    ],
    "flagged_fields": {},
    "missing_fields": [
      "purpose_and_intended_users.out_of_scope_uses",
      "ethical_and_legal_considerations.privacy_and_anonymity",
      "ethical_and_legal_considerations.consent_procedures",
      "ethical_and_legal_considerations.compliance_with_regulations"
    ],
    "card_info": {
      "created_at": "2026-03-17T13:56:24.159440",
      "llm": "deepseek-ai/DeepSeek-V3.2"
    }
  }
}