{
  "models": [
    {
      "model_id": "01-ai/yi-34b",
      "name": "Yi 34B",
      "developer": "01-ai",
      "scores": {
        "Mean win rate": 0.57,
        "NarrativeQA": 0.782,
        "NaturalQuestions (closed-book)": 0.443,
        "OpenbookQA": 0.92,
        "MMLU": 0.65,
        "MATH": 0.375,
        "GSM8K": 0.648,
        "LegalBench": 0.618,
        "MedQA": 0.656,
        "WMT 2014": 0.172
      }
    },
    {
      "model_id": "01-ai/yi-6b",
      "name": "Yi 6B",
      "developer": "01-ai",
      "scores": {
        "Mean win rate": 0.253,
        "NarrativeQA": 0.702,
        "NaturalQuestions (closed-book)": 0.31,
        "OpenbookQA": 0.8,
        "MMLU": 0.53,
        "MATH": 0.126,
        "GSM8K": 0.375,
        "LegalBench": 0.519,
        "MedQA": 0.497,
        "WMT 2014": 0.117
      }
    },
    {
      "model_id": "01-ai/yi-large-preview",
      "name": "Yi Large Preview",
      "developer": "01-ai",
      "scores": {
        "Mean win rate": 0.471,
        "NarrativeQA": 0.373,
        "NaturalQuestions (closed-book)": 0.428,
        "OpenbookQA": 0.946,
        "MMLU": 0.712,
        "MATH": 0.712,
        "GSM8K": 0.69,
        "LegalBench": 0.519,
        "MedQA": 0.66,
        "WMT 2014": 0.176
      }
    },
    {
      "model_id": "AlephAlpha/luminous-base",
      "name": "Luminous Base 13B",
      "developer": "AlephAlpha",
      "scores": {
        "Mean win rate": 0.041,
        "NarrativeQA": 0.633,
        "NaturalQuestions (closed-book)": 0.197,
        "OpenbookQA": 0.286,
        "MMLU": 0.243,
        "MATH": 0.026,
        "GSM8K": 0.028,
        "LegalBench": 0.332,
        "MedQA": 0.26,
        "WMT 2014": 0.066
      }
    },
    {
      "model_id": "AlephAlpha/luminous-extended",
      "name": "Luminous Extended 30B",
      "developer": "AlephAlpha",
      "scores": {
        "Mean win rate": 0.078,
        "NarrativeQA": 0.684,
        "NaturalQuestions (closed-book)": 0.253,
        "OpenbookQA": 0.272,
        "MMLU": 0.248,
        "MATH": 0.04,
        "GSM8K": 0.075,
        "LegalBench": 0.421,
        "MedQA": 0.276,
        "WMT 2014": 0.083
      }
    },
    {
      "model_id": "AlephAlpha/luminous-supreme",
      "name": "Luminous Supreme 70B",
      "developer": "AlephAlpha",
      "scores": {
        "Mean win rate": 0.145,
        "NarrativeQA": 0.743,
        "NaturalQuestions (closed-book)": 0.299,
        "OpenbookQA": 0.284,
        "MMLU": 0.316,
        "MATH": 0.078,
        "GSM8K": 0.137,
        "LegalBench": 0.452,
        "MedQA": 0.276,
        "WMT 2014": 0.102
      }
    },
    {
      "model_id": "ai21/j2-grande",
      "name": "Jurassic-2 Grande 17B",
      "developer": "ai21",
      "scores": {
        "Mean win rate": 0.172,
        "NarrativeQA": 0.744,
        "NaturalQuestions (closed-book)": 0.35,
        "OpenbookQA": 0.614,
        "MMLU": 0.471,
        "MATH": 0.064,
        "GSM8K": 0.159,
        "LegalBench": 0.468,
        "MedQA": 0.39,
        "WMT 2014": 0.102
      }
    },
    {
      "model_id": "ai21/j2-jumbo",
      "name": "Jurassic-2 Jumbo 178B",
      "developer": "ai21",
      "scores": {
        "Mean win rate": 0.215,
        "NarrativeQA": 0.728,
        "NaturalQuestions (closed-book)": 0.385,
        "OpenbookQA": 0.688,
        "MMLU": 0.483,
        "MATH": 0.103,
        "GSM8K": 0.239,
        "LegalBench": 0.533,
        "MedQA": 0.431,
        "WMT 2014": 0.114
      }
    },
    {
      "model_id": "ai21/jamba-1.5-large",
      "name": "Jamba 1.5 Large",
      "developer": "ai21",
      "scores": {
        "Mean win rate": 0.637,
        "NarrativeQA": 0.664,
        "NaturalQuestions (closed-book)": 0.394,
        "OpenbookQA": 0.948,
        "MMLU": 0.683,
        "MATH": 0.692,
        "GSM8K": 0.846,
        "LegalBench": 0.675,
        "MedQA": 0.698,
        "WMT 2014": 0.203
      }
    },
    {
      "model_id": "ai21/jamba-1.5-mini",
      "name": "Jamba 1.5 Mini",
      "developer": "ai21",
      "scores": {
        "Mean win rate": 0.414,
        "NarrativeQA": 0.746,
        "NaturalQuestions (closed-book)": 0.388,
        "OpenbookQA": 0.89,
        "MMLU": 0.582,
        "MATH": 0.318,
        "GSM8K": 0.691,
        "LegalBench": 0.503,
        "MedQA": 0.632,
        "WMT 2014": 0.179
      }
    },
    {
      "model_id": "ai21/jamba-instruct",
      "name": "Jamba Instruct",
      "developer": "ai21",
      "scores": {
        "Mean win rate": 0.287,
        "NarrativeQA": 0.658,
        "NaturalQuestions (closed-book)": 0.384,
        "OpenbookQA": 0.796,
        "MMLU": 0.582,
        "MATH": 0.38,
        "GSM8K": 0.67,
        "LegalBench": 0.54,
        "MedQA": 0.519,
        "WMT 2014": 0.164
      }
    },
    {
      "model_id": "allenai/olmo-7b",
      "name": "OLMo 7B",
      "developer": "allenai",
      "scores": {
        "Mean win rate": 0.052,
        "NarrativeQA": 0.597,
        "NaturalQuestions (closed-book)": 0.259,
        "OpenbookQA": 0.222,
        "MMLU": 0.305,
        "MATH": 0.029,
        "GSM8K": 0.044,
        "LegalBench": 0.341,
        "MedQA": 0.229,
        "WMT 2014": 0.097
      }
    },
    {
      "model_id": "amazon/nova-lite-v1:0",
      "name": "Amazon Nova Lite",
      "developer": "amazon",
      "scores": {
        "Mean win rate": 0.708,
        "NarrativeQA": 0.768,
        "NaturalQuestions (closed-book)": 0.352,
        "OpenbookQA": 0.928,
        "MMLU": 0.693,
        "MATH": 0.779,
        "GSM8K": 0.829,
        "LegalBench": 0.659,
        "MedQA": 0.696,
        "WMT 2014": 0.204
      }
    },
    {
      "model_id": "amazon/nova-micro-v1:0",
      "name": "Amazon Nova Micro",
      "developer": "amazon",
      "scores": {
        "Mean win rate": 0.524,
        "NarrativeQA": 0.744,
        "NaturalQuestions (closed-book)": 0.285,
        "OpenbookQA": 0.888,
        "MMLU": 0.64,
        "MATH": 0.76,
        "GSM8K": 0.794,
        "LegalBench": 0.615,
        "MedQA": 0.608,
        "WMT 2014": 0.192
      }
    },
    {
      "model_id": "amazon/nova-pro-v1:0",
      "name": "Amazon Nova Pro",
      "developer": "amazon",
      "scores": {
        "Mean win rate": 0.885,
        "NarrativeQA": 0.791,
        "NaturalQuestions (closed-book)": 0.405,
        "OpenbookQA": 0.96,
        "MMLU": 0.758,
        "MATH": 0.821,
        "GSM8K": 0.87,
        "LegalBench": 0.736,
        "MedQA": 0.811,
        "WMT 2014": 0.229
      }
    },
    {
      "model_id": "anthropic/claude-2.0",
      "name": "Claude 2.0",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.489,
        "NarrativeQA": 0.718,
        "NaturalQuestions (closed-book)": 0.428,
        "OpenbookQA": 0.862,
        "MMLU": 0.639,
        "MATH": 0.603,
        "GSM8K": 0.583,
        "LegalBench": 0.643,
        "MedQA": 0.652,
        "WMT 2014": 0.219
      }
    },
    {
      "model_id": "anthropic/claude-2.1",
      "name": "Claude 2.1",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.437,
        "NarrativeQA": 0.677,
        "NaturalQuestions (closed-book)": 0.375,
        "OpenbookQA": 0.872,
        "MMLU": 0.643,
        "MATH": 0.632,
        "GSM8K": 0.604,
        "LegalBench": 0.643,
        "MedQA": 0.644,
        "WMT 2014": 0.204
      }
    },
    {
      "model_id": "anthropic/claude-3-5-haiku-20241022",
      "name": "Claude 3.5 Haiku 20241022",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.531,
        "NarrativeQA": 0.763,
        "NaturalQuestions (closed-book)": 0.344,
        "OpenbookQA": 0.854,
        "MMLU": 0.671,
        "MATH": 0.872,
        "GSM8K": 0.815,
        "LegalBench": 0.631,
        "MedQA": 0.722,
        "WMT 2014": 0.135
      }
    },
    {
      "model_id": "anthropic/claude-3-5-sonnet-20240620",
      "name": "Claude 3.5 Sonnet 20240620",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.885,
        "NarrativeQA": 0.746,
        "NaturalQuestions (closed-book)": 0.502,
        "OpenbookQA": 0.972,
        "MMLU": 0.799,
        "MATH": 0.813,
        "GSM8K": 0.949,
        "LegalBench": 0.707,
        "MedQA": 0.825,
        "WMT 2014": 0.229
      }
    },
    {
      "model_id": "anthropic/claude-3-5-sonnet-20241022",
      "name": "Claude 3.5 Sonnet 20241022",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.846,
        "NarrativeQA": 0.77,
        "NaturalQuestions (closed-book)": 0.467,
        "OpenbookQA": 0.966,
        "MMLU": 0.809,
        "MATH": 0.904,
        "GSM8K": 0.956,
        "LegalBench": 0.647,
        "MedQA": 0.859,
        "WMT 2014": 0.226
      }
    },
    {
      "model_id": "anthropic/claude-3-haiku-20240307",
      "name": "Claude 3 Haiku 20240307",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.263,
        "NarrativeQA": 0.244,
        "NaturalQuestions (closed-book)": 0.144,
        "OpenbookQA": 0.838,
        "MMLU": 0.662,
        "MATH": 0.131,
        "GSM8K": 0.699,
        "LegalBench": 0.46,
        "MedQA": 0.702,
        "WMT 2014": 0.148
      }
    },
    {
      "model_id": "anthropic/claude-3-opus-20240229",
      "name": "Claude 3 Opus 20240229",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.683,
        "NarrativeQA": 0.351,
        "NaturalQuestions (closed-book)": 0.441,
        "OpenbookQA": 0.956,
        "MMLU": 0.768,
        "MATH": 0.76,
        "GSM8K": 0.924,
        "LegalBench": 0.662,
        "MedQA": 0.775,
        "WMT 2014": 0.24
      }
    },
    {
      "model_id": "anthropic/claude-3-sonnet-20240229",
      "name": "Claude 3 Sonnet 20240229",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.377,
        "NarrativeQA": 0.111,
        "NaturalQuestions (closed-book)": 0.028,
        "OpenbookQA": 0.918,
        "MMLU": 0.652,
        "MATH": 0.084,
        "GSM8K": 0.907,
        "LegalBench": 0.49,
        "MedQA": 0.684,
        "WMT 2014": 0.218
      }
    },
    {
      "model_id": "anthropic/claude-instant-1.2",
      "name": "Claude Instant 1.2",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.399,
        "NarrativeQA": 0.616,
        "NaturalQuestions (closed-book)": 0.343,
        "OpenbookQA": 0.844,
        "MMLU": 0.631,
        "MATH": 0.499,
        "GSM8K": 0.721,
        "LegalBench": 0.586,
        "MedQA": 0.559,
        "WMT 2014": 0.194
      }
    },
    {
      "model_id": "anthropic/claude-v1.3",
      "name": "Anthropic Claude v1.3",
      "developer": "Anthropic",
      "scores": {
        "Mean win rate": 0.518,
        "NarrativeQA": 0.723,
        "NaturalQuestions (closed-book)": 0.409,
        "OpenbookQA": 0.908,
        "MMLU": 0.631,
        "MATH": 0.54,
        "GSM8K": 0.784,
        "LegalBench": 0.629,
        "MedQA": 0.618,
        "WMT 2014": 0.219
      }
    },
    {
      "model_id": "cohere/command",
      "name": "Command",
      "developer": "cohere",
      "scores": {
        "Mean win rate": 0.327,
        "NarrativeQA": 0.749,
        "NaturalQuestions (closed-book)": 0.391,
        "OpenbookQA": 0.774,
        "MMLU": 0.525,
        "MATH": 0.236,
        "GSM8K": 0.452,
        "LegalBench": 0.578,
        "MedQA": 0.445,
        "WMT 2014": 0.088
      }
    },
    {
      "model_id": "cohere/command-light",
      "name": "Command Light",
      "developer": "cohere",
      "scores": {
        "Mean win rate": 0.105,
        "NarrativeQA": 0.629,
        "NaturalQuestions (closed-book)": 0.195,
        "OpenbookQA": 0.398,
        "MMLU": 0.386,
        "MATH": 0.098,
        "GSM8K": 0.149,
        "LegalBench": 0.397,
        "MedQA": 0.312,
        "WMT 2014": 0.023
      }
    },
    {
      "model_id": "cohere/command-r",
      "name": "Command R",
      "developer": "cohere",
      "scores": {
        "Mean win rate": 0.299,
        "NarrativeQA": 0.742,
        "NaturalQuestions (closed-book)": 0.352,
        "OpenbookQA": 0.782,
        "MMLU": 0.567,
        "MATH": 0.266,
        "GSM8K": 0.551,
        "LegalBench": 0.507,
        "MedQA": 0.555,
        "WMT 2014": 0.149
      }
    },
    {
      "model_id": "cohere/command-r-plus",
      "name": "Command R Plus",
      "developer": "cohere",
      "scores": {
        "Mean win rate": 0.441,
        "NarrativeQA": 0.735,
        "NaturalQuestions (closed-book)": 0.343,
        "OpenbookQA": 0.828,
        "MMLU": 0.59,
        "MATH": 0.403,
        "GSM8K": 0.738,
        "LegalBench": 0.672,
        "MedQA": 0.567,
        "WMT 2014": 0.203
      }
    },
    {
      "model_id": "databricks/dbrx-instruct",
      "name": "DBRX Instruct",
      "developer": "databricks",
      "scores": {
        "Mean win rate": 0.289,
        "NarrativeQA": 0.488,
        "NaturalQuestions (closed-book)": 0.284,
        "OpenbookQA": 0.91,
        "MMLU": 0.643,
        "MATH": 0.358,
        "GSM8K": 0.671,
        "LegalBench": 0.426,
        "MedQA": 0.694,
        "WMT 2014": 0.131
      }
    },
    {
      "model_id": "deepseek-ai/deepseek-llm-67b-chat",
      "name": "DeepSeek LLM Chat 67B",
      "developer": "deepseek-ai",
      "scores": {
        "Mean win rate": 0.488,
        "NarrativeQA": 0.581,
        "NaturalQuestions (closed-book)": 0.412,
        "OpenbookQA": 0.88,
        "MMLU": 0.641,
        "MATH": 0.615,
        "GSM8K": 0.795,
        "LegalBench": 0.637,
        "MedQA": 0.628,
        "WMT 2014": 0.186
      }
    },
    {
      "model_id": "deepseek-ai/deepseek-v3",
      "name": "DeepSeek v3",
      "developer": "deepseek-ai",
      "scores": {
        "Mean win rate": 0.908,
        "NarrativeQA": 0.796,
        "NaturalQuestions (closed-book)": 0.467,
        "OpenbookQA": 0.954,
        "MMLU": 0.803,
        "MATH": 0.912,
        "GSM8K": 0.94,
        "LegalBench": 0.718,
        "MedQA": 0.809,
        "WMT 2014": 0.209
      }
    },
    {
      "model_id": "google/gemini-1.0-pro-002",
      "name": "Gemini 1.0 Pro 002",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.422,
        "NarrativeQA": 0.751,
        "NaturalQuestions (closed-book)": 0.391,
        "OpenbookQA": 0.788,
        "MMLU": 0.534,
        "MATH": 0.665,
        "GSM8K": 0.816,
        "LegalBench": 0.475,
        "MedQA": 0.483,
        "WMT 2014": 0.194
      }
    },
    {
      "model_id": "google/gemini-1.5-flash-001",
      "name": "Gemini 1.5 Flash 001",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.667,
        "NarrativeQA": 0.783,
        "NaturalQuestions (closed-book)": 0.332,
        "OpenbookQA": 0.928,
        "MMLU": 0.703,
        "MATH": 0.753,
        "GSM8K": 0.785,
        "LegalBench": 0.661,
        "MedQA": 0.68,
        "WMT 2014": 0.225
      }
    },
    {
      "model_id": "google/gemini-1.5-flash-002",
      "name": "Gemini 1.5 Flash 002",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.573,
        "NarrativeQA": 0.746,
        "NaturalQuestions (closed-book)": 0.323,
        "OpenbookQA": 0.914,
        "MMLU": 0.679,
        "MATH": 0.908,
        "GSM8K": 0.328,
        "LegalBench": 0.67,
        "MedQA": 0.656,
        "WMT 2014": 0.212
      }
    },
    {
      "model_id": "google/gemini-1.5-pro-001",
      "name": "Gemini 1.5 Pro 001",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.739,
        "NarrativeQA": 0.783,
        "NaturalQuestions (closed-book)": 0.378,
        "OpenbookQA": 0.902,
        "MMLU": 0.772,
        "MATH": 0.825,
        "GSM8K": 0.836,
        "LegalBench": 0.757,
        "MedQA": 0.692,
        "WMT 2014": 0.189
      }
    },
    {
      "model_id": "google/gemini-1.5-pro-002",
      "name": "Gemini 1.5 Pro 002",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.842,
        "NarrativeQA": 0.756,
        "NaturalQuestions (closed-book)": 0.455,
        "OpenbookQA": 0.952,
        "MMLU": 0.795,
        "MATH": 0.92,
        "GSM8K": 0.817,
        "LegalBench": 0.747,
        "MedQA": 0.771,
        "WMT 2014": 0.231
      }
    },
    {
      "model_id": "google/gemini-2.0-flash-exp",
      "name": "Gemini 2.0 Flash Experimental",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.813,
        "NarrativeQA": 0.783,
        "NaturalQuestions (closed-book)": 0.443,
        "OpenbookQA": 0.946,
        "MMLU": 0.717,
        "MATH": 0.901,
        "GSM8K": 0.946,
        "LegalBench": 0.674,
        "MedQA": 0.73,
        "WMT 2014": 0.212
      }
    },
    {
      "model_id": "google/gemma-2-27b-it",
      "name": "Gemma 2 Instruct 27B",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.675,
        "NarrativeQA": 0.79,
        "NaturalQuestions (closed-book)": 0.353,
        "OpenbookQA": 0.918,
        "MMLU": 0.664,
        "MATH": 0.746,
        "GSM8K": 0.812,
        "LegalBench": 0.7,
        "MedQA": 0.684,
        "WMT 2014": 0.214
      }
    },
    {
      "model_id": "google/gemma-2-9b-it",
      "name": "Gemma 2 Instruct 9B",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.562,
        "NarrativeQA": 0.768,
        "NaturalQuestions (closed-book)": 0.328,
        "OpenbookQA": 0.91,
        "MMLU": 0.645,
        "MATH": 0.724,
        "GSM8K": 0.762,
        "LegalBench": 0.639,
        "MedQA": 0.63,
        "WMT 2014": 0.201
      }
    },
    {
      "model_id": "google/gemma-7b",
      "name": "Gemma 7B",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.336,
        "NarrativeQA": 0.752,
        "NaturalQuestions (closed-book)": 0.336,
        "OpenbookQA": 0.808,
        "MMLU": 0.571,
        "MATH": 0.5,
        "GSM8K": 0.559,
        "LegalBench": 0.581,
        "MedQA": 0.513,
        "WMT 2014": 0.187
      }
    },
    {
      "model_id": "google/text-bison@001",
      "name": "PaLM-2 Bison",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.526,
        "NarrativeQA": 0.718,
        "NaturalQuestions (closed-book)": 0.39,
        "OpenbookQA": 0.878,
        "MMLU": 0.608,
        "MATH": 0.421,
        "GSM8K": 0.61,
        "LegalBench": 0.645,
        "MedQA": 0.547,
        "WMT 2014": 0.241
      }
    },
    {
      "model_id": "google/text-unicorn@001",
      "name": "PaLM-2 Unicorn",
      "developer": "Google",
      "scores": {
        "Mean win rate": 0.644,
        "NarrativeQA": 0.583,
        "NaturalQuestions (closed-book)": 0.435,
        "OpenbookQA": 0.938,
        "MMLU": 0.702,
        "MATH": 0.674,
        "GSM8K": 0.831,
        "LegalBench": 0.677,
        "MedQA": 0.684,
        "WMT 2014": 0.26
      }
    },
    {
      "model_id": "meta/LLaMA-65B",
      "name": "LLaMA 65B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.345,
        "NarrativeQA": 0.755,
        "NaturalQuestions (closed-book)": 0.433,
        "OpenbookQA": 0.754,
        "MMLU": 0.584,
        "MATH": 0.257,
        "GSM8K": 0.489,
        "LegalBench": 0.48,
        "MedQA": 0.507,
        "WMT 2014": 0.189
      }
    },
    {
      "model_id": "meta/llama-2-13b",
      "name": "Llama 2 13B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.233,
        "NarrativeQA": 0.741,
        "NaturalQuestions (closed-book)": 0.371,
        "OpenbookQA": 0.634,
        "MMLU": 0.505,
        "MATH": 0.102,
        "GSM8K": 0.266,
        "LegalBench": 0.591,
        "MedQA": 0.392,
        "WMT 2014": 0.167
      }
    },
    {
      "model_id": "meta/llama-2-70b",
      "name": "Llama 2 70B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.482,
        "NarrativeQA": 0.763,
        "NaturalQuestions (closed-book)": 0.46,
        "OpenbookQA": 0.838,
        "MMLU": 0.58,
        "MATH": 0.323,
        "GSM8K": 0.567,
        "LegalBench": 0.673,
        "MedQA": 0.618,
        "WMT 2014": 0.196
      }
    },
    {
      "model_id": "meta/llama-2-7b",
      "name": "Llama 2 7B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.152,
        "NarrativeQA": 0.686,
        "NaturalQuestions (closed-book)": 0.333,
        "OpenbookQA": 0.544,
        "MMLU": 0.425,
        "MATH": 0.097,
        "GSM8K": 0.154,
        "LegalBench": 0.502,
        "MedQA": 0.392,
        "WMT 2014": 0.144
      }
    },
    {
      "model_id": "meta/llama-3-70b",
      "name": "Llama 3 70B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.793,
        "NarrativeQA": 0.798,
        "NaturalQuestions (closed-book)": 0.475,
        "OpenbookQA": 0.934,
        "MMLU": 0.695,
        "MATH": 0.663,
        "GSM8K": 0.805,
        "LegalBench": 0.733,
        "MedQA": 0.777,
        "WMT 2014": 0.225
      }
    },
    {
      "model_id": "meta/llama-3-8b",
      "name": "Llama 3 8B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.387,
        "NarrativeQA": 0.754,
        "NaturalQuestions (closed-book)": 0.378,
        "OpenbookQA": 0.766,
        "MMLU": 0.602,
        "MATH": 0.391,
        "GSM8K": 0.499,
        "LegalBench": 0.637,
        "MedQA": 0.581,
        "WMT 2014": 0.183
      }
    },
    {
      "model_id": "meta/llama-3.1-405b-instruct-turbo",
      "name": "Llama 3.1 Instruct Turbo 405B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.854,
        "NarrativeQA": 0.749,
        "NaturalQuestions (closed-book)": 0.456,
        "OpenbookQA": 0.94,
        "MMLU": 0.759,
        "MATH": 0.827,
        "GSM8K": 0.949,
        "LegalBench": 0.707,
        "MedQA": 0.805,
        "WMT 2014": 0.238
      }
    },
    {
      "model_id": "meta/llama-3.1-70b-instruct-turbo",
      "name": "Llama 3.1 Instruct Turbo 70B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.808,
        "NarrativeQA": 0.772,
        "NaturalQuestions (closed-book)": 0.452,
        "OpenbookQA": 0.938,
        "MMLU": 0.709,
        "MATH": 0.783,
        "GSM8K": 0.938,
        "LegalBench": 0.687,
        "MedQA": 0.769,
        "WMT 2014": 0.223
      }
    },
    {
      "model_id": "meta/llama-3.1-8b-instruct-turbo",
      "name": "Llama 3.1 Instruct Turbo 8B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.303,
        "NarrativeQA": 0.756,
        "NaturalQuestions (closed-book)": 0.209,
        "OpenbookQA": 0.74,
        "MMLU": 0.5,
        "MATH": 0.703,
        "GSM8K": 0.798,
        "LegalBench": 0.342,
        "MedQA": 0.245,
        "WMT 2014": 0.181
      }
    },
    {
      "model_id": "meta/llama-3.2-11b-vision-instruct-turbo",
      "name": "Llama 3.2 Vision Instruct Turbo 11B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.325,
        "NarrativeQA": 0.756,
        "NaturalQuestions (closed-book)": 0.234,
        "OpenbookQA": 0.724,
        "MMLU": 0.511,
        "MATH": 0.739,
        "GSM8K": 0.823,
        "LegalBench": 0.435,
        "MedQA": 0.27,
        "WMT 2014": 0.179
      }
    },
    {
      "model_id": "meta/llama-3.2-90b-vision-instruct-turbo",
      "name": "Llama 3.2 Vision Instruct Turbo 90B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.819,
        "NarrativeQA": 0.777,
        "NaturalQuestions (closed-book)": 0.457,
        "OpenbookQA": 0.942,
        "MMLU": 0.703,
        "MATH": 0.791,
        "GSM8K": 0.936,
        "LegalBench": 0.68,
        "MedQA": 0.769,
        "WMT 2014": 0.224
      }
    },
    {
      "model_id": "meta/llama-3.3-70b-instruct-turbo",
      "name": "Llama 3.3 Instruct Turbo 70B",
      "developer": "Meta",
      "scores": {
        "Mean win rate": 0.812,
        "NarrativeQA": 0.791,
        "NaturalQuestions (closed-book)": 0.431,
        "OpenbookQA": 0.928,
        "MMLU": 0.7,
        "MATH": 0.808,
        "GSM8K": 0.942,
        "LegalBench": 0.725,
        "MedQA": 0.761,
        "WMT 2014": 0.219
      }
    },
    {
      "model_id": "microsoft/phi-2",
      "name": "Phi-2",
      "developer": "microsoft",
      "scores": {
        "Mean win rate": 0.169,
        "NarrativeQA": 0.703,
        "NaturalQuestions (closed-book)": 0.155,
        "OpenbookQA": 0.798,
        "MMLU": 0.518,
        "MATH": 0.255,
        "GSM8K": 0.581,
        "LegalBench": 0.334,
        "MedQA": 0.41,
        "WMT 2014": 0.038
      }
    },
    {
      "model_id": "microsoft/phi-3-medium-4k-instruct",
      "name": "Phi-3 14B",
      "developer": "microsoft",
      "scores": {
        "Mean win rate": 0.509,
        "NarrativeQA": 0.724,
        "NaturalQuestions (closed-book)": 0.278,
        "OpenbookQA": 0.916,
        "MMLU": 0.675,
        "MATH": 0.611,
        "GSM8K": 0.878,
        "LegalBench": 0.593,
        "MedQA": 0.696,
        "WMT 2014": 0.17
      }
    },
    {
      "model_id": "microsoft/phi-3-small-8k-instruct",
      "name": "Phi-3 7B",
      "developer": "microsoft",
      "scores": {
        "Mean win rate": 0.473,
        "NarrativeQA": 0.754,
        "NaturalQuestions (closed-book)": 0.324,
        "OpenbookQA": 0.912,
        "MMLU": 0.659,
        "MATH": 0.703,
        "GSM8K": -1.0,
        "LegalBench": 0.584,
        "MedQA": 0.672,
        "WMT 2014": 0.154
      }
    },
    {
      "model_id": "mistralai/mistral-7b-instruct-v0.3",
      "name": "Mistral Instruct v0.3 7B",
      "developer": "mistralai",
      "scores": {
        "Mean win rate": 0.196,
        "NarrativeQA": 0.716,
        "NaturalQuestions (closed-book)": 0.253,
        "OpenbookQA": 0.79,
        "MMLU": 0.51,
        "MATH": 0.289,
        "GSM8K": 0.538,
        "LegalBench": 0.331,
        "MedQA": 0.517,
        "WMT 2014": 0.142
      }
    },
    {
      "model_id": "mistralai/mistral-7b-v0.1",
      "name": "Mistral v0.1 7B",
      "developer": "mistralai",
      "scores": {
        "Mean win rate": 0.292,
        "NarrativeQA": 0.716,
        "NaturalQuestions (closed-book)": 0.367,
        "OpenbookQA": 0.776,
        "MMLU": 0.584,
        "MATH": 0.297,
        "GSM8K": 0.377,
        "LegalBench": 0.58,
        "MedQA": 0.525,
        "WMT 2014": 0.16
      }
    },
    {
      "model_id": "mistralai/mistral-large-2402",
      "name": "Mistral Large 2402",
      "developer": "mistralai",
      "scores": {
        "Mean win rate": 0.328,
        "NarrativeQA": 0.454,
        "NaturalQuestions (closed-book)": 0.311,
        "OpenbookQA": 0.894,
        "MMLU": 0.638,
        "MATH": 0.75,
        "GSM8K": 0.694,
        "LegalBench": 0.479,
        "MedQA": 0.499,
        "WMT 2014": 0.182
      }
    },
    {
      "model_id": "mistralai/mistral-large-2407",
      "name": "Mistral Large 2 2407",
      "developer": "mistralai",
      "scores": {
        "Mean win rate": 0.744,
        "NarrativeQA": 0.779,
        "NaturalQuestions (closed-book)": 0.453,
        "OpenbookQA": 0.932,
        "MMLU": 0.725,
        "MATH": 0.677,
        "GSM8K": 0.912,
        "LegalBench": 0.646,
        "MedQA": 0.775,
        "WMT 2014": 0.192
      }
    },
    {
      "model_id": "mistralai/mistral-medium-2312",
      "name": "Mistral Medium 2312",
      "developer": "mistralai",
      "scores": {
        "Mean win rate": 0.268,
        "NarrativeQA": 0.449,
        "NaturalQuestions (closed-book)": 0.29,
        "OpenbookQA": 0.83,
        "MMLU": 0.618,
        "MATH": 0.565,
        "GSM8K": 0.706,
        "LegalBench": 0.452,
        "MedQA": 0.61,
        "WMT 2014": 0.169
      }
    },
    {
      "model_id": "mistralai/mistral-small-2402",
      "name": "Mistral Small 2402",
      "developer": "mistralai",
      "scores": {
        "Mean win rate": 0.288,
        "NarrativeQA": 0.519,
        "NaturalQuestions (closed-book)": 0.304,
        "OpenbookQA": 0.862,
        "MMLU": 0.593,
        "MATH": 0.621,
        "GSM8K": 0.734,
        "LegalBench": 0.389,
        "MedQA": 0.616,
        "WMT 2014": 0.169
      }
    },
    {
      "model_id": "mistralai/mixtral-8x22b",
      "name": "Mixtral 8x22B",
      "developer": "mistralai",
      "scores": {
        "Mean win rate": 0.705,
        "NarrativeQA": 0.779,
        "NaturalQuestions (closed-book)": 0.478,
        "OpenbookQA": 0.882,
        "MMLU": 0.701,
        "MATH": 0.656,
        "GSM8K": 0.8,
        "LegalBench": 0.708,
        "MedQA": 0.704,
        "WMT 2014": 0.209
      }
    },
    {
      "model_id": "mistralai/mixtral-8x7b-32kseqlen",
      "name": "Mixtral 8x7B 32K seqlen",
      "developer": "mistralai",
      "scores": {
        "Mean win rate": 0.51,
        "NarrativeQA": 0.767,
        "NaturalQuestions (closed-book)": 0.427,
        "OpenbookQA": 0.868,
        "MMLU": 0.649,
        "MATH": 0.494,
        "GSM8K": 0.622,
        "LegalBench": 0.63,
        "MedQA": 0.652,
        "WMT 2014": 0.19
      }
    },
    {
      "model_id": "mistralai/open-mistral-nemo-2407",
      "name": "Mistral NeMo 2402",
      "developer": "mistralai",
      "scores": {
        "Mean win rate": 0.333,
        "NarrativeQA": 0.731,
        "NaturalQuestions (closed-book)": 0.265,
        "OpenbookQA": 0.822,
        "MMLU": 0.604,
        "MATH": 0.668,
        "GSM8K": 0.782,
        "LegalBench": 0.415,
        "MedQA": 0.59,
        "WMT 2014": 0.177
      }
    },
    {
      "model_id": "openai/gpt-3.5-turbo-0613",
      "name": "GPT-3.5 Turbo 0613",
      "developer": "OpenAI",
      "scores": {
        "Mean win rate": 0.358,
        "NarrativeQA": 0.655,
        "NaturalQuestions (closed-book)": 0.335,
        "OpenbookQA": 0.838,
        "MMLU": 0.614,
        "MATH": 0.667,
        "GSM8K": 0.501,
        "LegalBench": 0.528,
        "MedQA": 0.622,
        "WMT 2014": 0.187
      }
    },
    {
      "model_id": "openai/gpt-4-0613",
      "name": "GPT-4 0613",
      "developer": "OpenAI",
      "scores": {
        "Mean win rate": 0.867,
        "NarrativeQA": 0.768,
        "NaturalQuestions (closed-book)": 0.457,
        "OpenbookQA": 0.96,
        "MMLU": 0.735,
        "MATH": 0.802,
        "GSM8K": 0.932,
        "LegalBench": 0.713,
        "MedQA": 0.815,
        "WMT 2014": 0.211
      }
    },
    {
      "model_id": "openai/gpt-4-1106-preview",
      "name": "GPT-4 Turbo 1106 preview",
      "developer": "OpenAI",
      "scores": {
        "Mean win rate": 0.698,
        "NarrativeQA": 0.727,
        "NaturalQuestions (closed-book)": 0.435,
        "OpenbookQA": 0.95,
        "MMLU": 0.699,
        "MATH": 0.857,
        "GSM8K": 0.668,
        "LegalBench": 0.626,
        "MedQA": 0.817,
        "WMT 2014": 0.205
      }
    },
    {
      "model_id": "openai/gpt-4-turbo-2024-04-09",
      "name": "GPT-4 Turbo 2024-04-09",
      "developer": "OpenAI",
      "scores": {
        "Mean win rate": 0.864,
        "NarrativeQA": 0.761,
        "NaturalQuestions (closed-book)": 0.482,
        "OpenbookQA": 0.97,
        "MMLU": 0.711,
        "MATH": 0.833,
        "GSM8K": 0.824,
        "LegalBench": 0.727,
        "MedQA": 0.783,
        "WMT 2014": 0.218
      }
    },
    {
      "model_id": "openai/gpt-4o-2024-05-13",
      "name": "GPT-4o 2024-05-13",
      "developer": "OpenAI",
      "scores": {
        "Mean win rate": 0.938,
        "NarrativeQA": 0.804,
        "NaturalQuestions (closed-book)": 0.501,
        "OpenbookQA": 0.966,
        "MMLU": 0.748,
        "MATH": 0.829,
        "GSM8K": 0.905,
        "LegalBench": 0.733,
        "MedQA": 0.857,
        "WMT 2014": 0.231
      }
    },
    {
      "model_id": "openai/gpt-4o-2024-08-06",
      "name": "GPT-4o 2024-08-06",
      "developer": "OpenAI",
      "scores": {
        "Mean win rate": 0.928,
        "NarrativeQA": 0.795,
        "NaturalQuestions (closed-book)": 0.496,
        "OpenbookQA": 0.968,
        "MMLU": 0.738,
        "MATH": 0.853,
        "GSM8K": 0.909,
        "LegalBench": 0.721,
        "MedQA": 0.863,
        "WMT 2014": 0.225
      }
    },
    {
      "model_id": "openai/gpt-4o-mini-2024-07-18",
      "name": "GPT-4o mini 2024-07-18",
      "developer": "OpenAI",
      "scores": {
        "Mean win rate": 0.701,
        "NarrativeQA": 0.768,
        "NaturalQuestions (closed-book)": 0.386,
        "OpenbookQA": 0.92,
        "MMLU": 0.668,
        "MATH": 0.802,
        "GSM8K": 0.843,
        "LegalBench": 0.653,
        "MedQA": 0.748,
        "WMT 2014": 0.206
      }
    },
    {
      "model_id": "openai/text-davinci-002",
      "name": "GPT-3.5 text-davinci-002",
      "developer": "OpenAI",
      "scores": {
        "Mean win rate": 0.336,
        "NarrativeQA": 0.719,
        "NaturalQuestions (closed-book)": 0.394,
        "OpenbookQA": 0.796,
        "MMLU": 0.568,
        "MATH": 0.428,
        "GSM8K": 0.479,
        "LegalBench": 0.58,
        "MedQA": 0.525,
        "WMT 2014": 0.174
      }
    },
    {
      "model_id": "openai/text-davinci-003",
      "name": "GPT-3.5 text-davinci-003",
      "developer": "OpenAI",
      "scores": {
        "Mean win rate": 0.439,
        "NarrativeQA": 0.731,
        "NaturalQuestions (closed-book)": 0.413,
        "OpenbookQA": 0.828,
        "MMLU": 0.555,
        "MATH": 0.449,
        "GSM8K": 0.615,
        "LegalBench": 0.622,
        "MedQA": 0.531,
        "WMT 2014": 0.191
      }
    },
    {
      "model_id": "qwen/qwen1.5-110b-chat",
      "name": "Qwen1.5 Chat 110B",
      "developer": "qwen",
      "scores": {
        "Mean win rate": 0.55,
        "NarrativeQA": 0.721,
        "NaturalQuestions (closed-book)": 0.35,
        "OpenbookQA": 0.922,
        "MMLU": 0.704,
        "MATH": 0.568,
        "GSM8K": 0.815,
        "LegalBench": 0.624,
        "MedQA": 0.64,
        "WMT 2014": 0.192
      }
    },
    {
      "model_id": "qwen/qwen1.5-14b",
      "name": "Qwen1.5 14B",
      "developer": "qwen",
      "scores": {
        "Mean win rate": 0.425,
        "NarrativeQA": 0.711,
        "NaturalQuestions (closed-book)": 0.3,
        "OpenbookQA": 0.862,
        "MMLU": 0.626,
        "MATH": 0.686,
        "GSM8K": 0.693,
        "LegalBench": 0.593,
        "MedQA": 0.515,
        "WMT 2014": 0.178
      }
    },
    {
      "model_id": "qwen/qwen1.5-32b",
      "name": "Qwen1.5 32B",
      "developer": "qwen",
      "scores": {
        "Mean win rate": 0.546,
        "NarrativeQA": 0.589,
        "NaturalQuestions (closed-book)": 0.353,
        "OpenbookQA": 0.932,
        "MMLU": 0.628,
        "MATH": 0.733,
        "GSM8K": 0.773,
        "LegalBench": 0.636,
        "MedQA": 0.656,
        "WMT 2014": 0.193
      }
    },
    {
      "model_id": "qwen/qwen1.5-72b",
      "name": "Qwen1.5 72B",
      "developer": "qwen",
      "scores": {
        "Mean win rate": 0.608,
        "NarrativeQA": 0.601,
        "NaturalQuestions (closed-book)": 0.417,
        "OpenbookQA": 0.93,
        "MMLU": 0.647,
        "MATH": 0.683,
        "GSM8K": 0.799,
        "LegalBench": 0.694,
        "MedQA": 0.67,
        "WMT 2014": 0.201
      }
    },
    {
      "model_id": "qwen/qwen1.5-7b",
      "name": "Qwen1.5 7B",
      "developer": "qwen",
      "scores": {
        "Mean win rate": 0.275,
        "NarrativeQA": 0.448,
        "NaturalQuestions (closed-book)": 0.27,
        "OpenbookQA": 0.806,
        "MMLU": 0.569,
        "MATH": 0.561,
        "GSM8K": 0.6,
        "LegalBench": 0.523,
        "MedQA": 0.479,
        "WMT 2014": 0.153
      }
    },
    {
      "model_id": "qwen/qwen2-72b-instruct",
      "name": "Qwen2 Instruct 72B",
      "developer": "qwen",
      "scores": {
        "Mean win rate": 0.77,
        "NarrativeQA": 0.727,
        "NaturalQuestions (closed-book)": 0.39,
        "OpenbookQA": 0.954,
        "MMLU": 0.769,
        "MATH": 0.79,
        "GSM8K": 0.92,
        "LegalBench": 0.712,
        "MedQA": 0.746,
        "WMT 2014": 0.207
      }
    },
    {
      "model_id": "qwen/qwen2.5-72b-instruct-turbo",
      "name": "Qwen2.5 Instruct Turbo 72B",
      "developer": "qwen",
      "scores": {
        "Mean win rate": 0.745,
        "NarrativeQA": 0.745,
        "NaturalQuestions (closed-book)": 0.359,
        "OpenbookQA": 0.962,
        "MMLU": 0.77,
        "MATH": 0.884,
        "GSM8K": 0.9,
        "LegalBench": 0.74,
        "MedQA": 0.753,
        "WMT 2014": 0.207
      }
    },
    {
      "model_id": "qwen/qwen2.5-7b-instruct-turbo",
      "name": "Qwen2.5 Instruct Turbo 7B",
      "developer": "qwen",
      "scores": {
        "Mean win rate": 0.488,
        "NarrativeQA": 0.742,
        "NaturalQuestions (closed-book)": 0.205,
        "OpenbookQA": 0.862,
        "MMLU": 0.658,
        "MATH": 0.835,
        "GSM8K": 0.83,
        "LegalBench": 0.632,
        "MedQA": 0.6,
        "WMT 2014": 0.155
      }
    },
    {
      "model_id": "snowflake/snowflake-arctic-instruct",
      "name": "Arctic Instruct",
      "developer": "snowflake",
      "scores": {
        "Mean win rate": 0.338,
        "NarrativeQA": 0.654,
        "NaturalQuestions (closed-book)": 0.39,
        "OpenbookQA": 0.828,
        "MMLU": 0.575,
        "MATH": 0.519,
        "GSM8K": 0.768,
        "LegalBench": 0.588,
        "MedQA": 0.581,
        "WMT 2014": 0.172
      }
    },
    {
      "model_id": "tiiuae/falcon-40b",
      "name": "Falcon 40B",
      "developer": "tiiuae",
      "scores": {
        "Mean win rate": 0.217,
        "NarrativeQA": 0.671,
        "NaturalQuestions (closed-book)": 0.392,
        "OpenbookQA": 0.662,
        "MMLU": 0.507,
        "MATH": 0.128,
        "GSM8K": 0.267,
        "LegalBench": 0.442,
        "MedQA": 0.419,
        "WMT 2014": 0.162
      }
    },
    {
      "model_id": "tiiuae/falcon-7b",
      "name": "Falcon 7B",
      "developer": "tiiuae",
      "scores": {
        "Mean win rate": 0.064,
        "NarrativeQA": 0.621,
        "NaturalQuestions (closed-book)": 0.285,
        "OpenbookQA": 0.26,
        "MMLU": 0.288,
        "MATH": 0.044,
        "GSM8K": 0.055,
        "LegalBench": 0.346,
        "MedQA": 0.254,
        "WMT 2014": 0.094
      }
    },
    {
      "model_id": "upstage/solar-pro-241126",
      "name": "Solar Pro",
      "developer": "upstage",
      "scores": {
        "Mean win rate": 0.602,
        "NarrativeQA": 0.753,
        "NaturalQuestions (closed-book)": 0.297,
        "OpenbookQA": 0.922,
        "MMLU": 0.679,
        "MATH": 0.567,
        "GSM8K": 0.871,
        "LegalBench": 0.67,
        "MedQA": 0.698,
        "WMT 2014": 0.169
      }
    },
    {
      "model_id": "writer/palmyra-x-004",
      "name": "Palmyra-X-004",
      "developer": "writer",
      "scores": {
        "Mean win rate": 0.808,
        "NarrativeQA": 0.773,
        "NaturalQuestions (closed-book)": 0.457,
        "OpenbookQA": 0.926,
        "MMLU": 0.739,
        "MATH": 0.767,
        "GSM8K": 0.905,
        "LegalBench": 0.73,
        "MedQA": 0.775,
        "WMT 2014": 0.203
      }
    },
    {
      "model_id": "writer/palmyra-x-v2",
      "name": "Palmyra X V2 33B",
      "developer": "writer",
      "scores": {
        "Mean win rate": 0.589,
        "NarrativeQA": 0.752,
        "NaturalQuestions (closed-book)": 0.428,
        "OpenbookQA": 0.878,
        "MMLU": 0.621,
        "MATH": 0.58,
        "GSM8K": 0.735,
        "LegalBench": 0.644,
        "MedQA": 0.598,
        "WMT 2014": 0.239
      }
    },
    {
      "model_id": "writer/palmyra-x-v3",
      "name": "Palmyra X V3 72B",
      "developer": "writer",
      "scores": {
        "Mean win rate": 0.679,
        "NarrativeQA": 0.706,
        "NaturalQuestions (closed-book)": 0.407,
        "OpenbookQA": 0.938,
        "MMLU": 0.702,
        "MATH": 0.723,
        "GSM8K": 0.831,
        "LegalBench": 0.709,
        "MedQA": 0.684,
        "WMT 2014": 0.262
      }
    }
  ]
}