data/benchmarks/livecodebenchpro.json

{
  "models": [
    {
      "model_id": "alibaba/qwen3-235b-a22b-thinking-2507",
      "name": "qwen3-235b-a22b-thinking-2507",
      "developer": "Alibaba",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.1267605633802817,
        "Easy Problems": 0.7605633802816901
      }
    },
    {
      "model_id": "alibaba/qwen3-30b-a3b",
      "name": "qwen3-30b-a3b",
      "developer": "Alibaba",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.028169014084507043,
        "Easy Problems": 0.5774647887323944
      }
    },
    {
      "model_id": "alibaba/qwen3-max",
      "name": "alibaba/qwen3-max",
      "developer": "Alibaba",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.04225352112676056,
        "Easy Problems": 0.36619718309859156
      }
    },
    {
      "model_id": "alibaba/qwen3-next-80b-a3b-thinking",
      "name": "qwen3-next-80b-a3b-thinking",
      "developer": "Alibaba",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.14084507042253522,
        "Easy Problems": 0.7464788732394366
      }
    },
    {
      "model_id": "aliyun/qwen3-next-80b-a3b-thinking",
      "name": "qwen3-next-80b-a3b-thinking",
      "developer": "aliyun",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.0704,
        "Easy Problems": 0.6901
      }
    },
    {
      "model_id": "anthropic/claude-3-7-sonnet-20250219",
      "name": "claude-3-7-sonnet-20250219",
      "developer": "Anthropic",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.0,
        "Easy Problems": 0.28169014084507044
      }
    },
    {
      "model_id": "anthropic/claude-3.7-sonnet",
      "name": "anthropic/claude-3.7-sonnet",
      "developer": "Anthropic",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.014084507042253521,
        "Easy Problems": 0.15492957746478872
      }
    },
    {
      "model_id": "anthropic/claude-sonnet-4-5-20250929",
      "name": "claude-sonnet-4-5-20250929",
      "developer": "Anthropic",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.0,
        "Easy Problems": 0.5352
      }
    },
    {
      "model_id": "ark/ep-20250603132404-cgpjm",
      "name": "ep-20250603132404-cgpjm",
      "developer": "ark",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.0141,
        "Easy Problems": 0.507
      }
    },
    {
      "model_id": "bytedance/doubao-seed-1-6-thinking-250615",
      "name": "doubao-seed-1-6-thinking-250615",
      "developer": "ByteDance",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.07042253521126761,
        "Easy Problems": 0.5774647887323944
      }
    },
    {
      "model_id": "deepseek/chat-v3-0324",
      "name": "deepseek/chat-v3-0324",
      "developer": "DeepSeek",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.0,
        "Easy Problems": 0.19718309859154928
      }
    },
    {
      "model_id": "deepseek/ep-20250214004308-p7n89",
      "name": "ep-20250214004308-p7n89",
      "developer": "DeepSeek",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.014084507042253521,
        "Easy Problems": 0.4225352112676056
      }
    },
    {
      "model_id": "deepseek/ep-20250228232227-z44x5",
      "name": "ep-20250228232227-z44x5",
      "developer": "DeepSeek",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.0,
        "Easy Problems": 0.1267605633802817
      }
    },
    {
      "model_id": "deepseek/ep-20250603132404-cgpjm",
      "name": "ep-20250603132404-cgpjm",
      "developer": "DeepSeek",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.08450704225352113,
        "Easy Problems": 0.5774647887323944
      }
    },
    {
      "model_id": "google/gemini-2.5-flash",
      "name": "Gemini 2.5 Flash",
      "developer": "Google",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.028169014084507043,
        "Easy Problems": 0.38028169014084506
      }
    },
    {
      "model_id": "google/gemini-2.5-pro",
      "name": "Gemini 2.5 Pro",
      "developer": "Google",
      "scores": {
        "Hard Problems": 0.014084507042253521,
        "Medium Problems": 0.2112676056338028,
        "Easy Problems": 0.7183098591549296
      }
    },
    {
      "model_id": "kuaishou/kwaipilot-40b-0604",
      "name": "kwaipilot-40b-0604",
      "developer": "Kuaishou",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.07042253521126761,
        "Easy Problems": 0.056338028169014086
      }
    },
    {
      "model_id": "meta/llama-4-maverick",
      "name": "meta/llama-4-maverick",
      "developer": "Meta",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.0,
        "Easy Problems": 0.09859154929577464
      }
    },
    {
      "model_id": "openai/gpt-4.1",
      "name": "openai/gpt-4.1",
      "developer": "OpenAI",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.0,
        "Easy Problems": 0.19718309859154928
      }
    },
    {
      "model_id": "openai/gpt-4o-2024-11-20",
      "name": "GPT-4o 2024-11-20",
      "developer": "OpenAI",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.0,
        "Easy Problems": 0.07042253521126761
      }
    },
    {
      "model_id": "openai/gpt-5-2025-08-07",
      "name": "gpt-5-2025-08-07",
      "developer": "OpenAI",
      "scores": {
        "Hard Problems": 0.0423,
        "Medium Problems": 0.4085,
        "Easy Problems": 0.9014
      }
    },
    {
      "model_id": "openai/gpt-5.2-2025-12-11",
      "name": "gpt-5.2-2025-12-11",
      "developer": "OpenAI",
      "scores": {
        "Hard Problems": 0.1594,
        "Medium Problems": 0.5211,
        "Easy Problems": 0.9014
      }
    },
    {
      "model_id": "openai/gpt-oss-120b",
      "name": "GPT-OSS-120B",
      "developer": "OpenAI",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.11267605633802817,
        "Easy Problems": 0.6619718309859155
      }
    },
    {
      "model_id": "openai/gpt-oss-20b",
      "name": "GPT-OSS-20B",
      "developer": "OpenAI",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.056338028169014086,
        "Easy Problems": 0.5070422535211268
      }
    },
    {
      "model_id": "openai/o3-2025-04-16",
      "name": "o3-2025-04-16",
      "developer": "OpenAI",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.22535211267605634,
        "Easy Problems": 0.7183098591549296
      }
    },
    {
      "model_id": "openai/o4-mini-2025-04-16",
      "name": "o4-mini-2025-04-16",
      "developer": "OpenAI",
      "scores": {
        "Hard Problems": 0.0143,
        "Medium Problems": 0.2923,
        "Easy Problems": 0.8571
      }
    },
    {
      "model_id": "z-ai/glm-4.5",
      "name": "z-ai/glm-4.5",
      "developer": "Z.ai",
      "scores": {
        "Hard Problems": 0.0,
        "Medium Problems": 0.028169014084507043,
        "Easy Problems": 0.1267605633802817
      }
    }
  ]
}