[
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "mawps",
    "ok": true,
    "gpu": 7
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "math_counting_easy",
    "ok": true,
    "gpu": 7
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "mbpp_sanitized",
    "ok": true,
    "gpu": 7
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "arc_easy",
    "ok": true,
    "gpu": 7
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "mmlu_high_school_biology",
    "ok": true,
    "gpu": 7
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "math_algebra_medium",
    "ok": true,
    "gpu": 7
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "mmlu_college_chemistry",
    "ok": true,
    "gpu": 7
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "gsm8k",
    "ok": true,
    "gpu": 1
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "multiarith",
    "ok": true,
    "gpu": 1
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "mbpp",
    "ok": true,
    "gpu": 1
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "conala_curated",
    "ok": true,
    "gpu": 1
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "openbookqa",
    "ok": true,
    "gpu": 1
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "mmlu_high_school_physics",
    "ok": true,
    "gpu": 1
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "humaneval_plus",
    "ok": true,
    "gpu": 1
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "mawps",
    "ok": true,
    "gpu": 6
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "math_counting_easy",
    "ok": true,
    "gpu": 6
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "mbpp_sanitized",
    "ok": true,
    "gpu": 6
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "arc_easy",
    "ok": true,
    "gpu": 6
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "mmlu_high_school_biology",
    "ok": true,
    "gpu": 6
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "math_algebra_medium",
    "ok": true,
    "gpu": 6
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "mmlu_college_chemistry",
    "ok": true,
    "gpu": 6
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "gsm8k",
    "ok": true,
    "gpu": 0
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "multiarith",
    "ok": true,
    "gpu": 0
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "mbpp",
    "ok": true,
    "gpu": 0
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "conala_curated",
    "ok": true,
    "gpu": 0
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "openbookqa",
    "ok": true,
    "gpu": 0
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "mmlu_high_school_physics",
    "ok": true,
    "gpu": 0
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "humaneval_plus",
    "ok": true,
    "gpu": 0
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "asdiv",
    "ok": true,
    "gpu": 5
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "math_algebra_easy",
    "ok": true,
    "gpu": 5
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "codealpaca_mini",
    "ok": true,
    "gpu": 5
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "sciq",
    "ok": true,
    "gpu": 5
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "mmlu_elementary_math",
    "ok": true,
    "gpu": 5
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "gsm_hard",
    "ok": true,
    "gpu": 5
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "arc_challenge",
    "ok": true,
    "gpu": 5
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "asdiv",
    "ok": true,
    "gpu": 4
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "math_algebra_easy",
    "ok": true,
    "gpu": 4
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "codealpaca_mini",
    "ok": true,
    "gpu": 4
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "sciq",
    "ok": true,
    "gpu": 4
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "mmlu_elementary_math",
    "ok": true,
    "gpu": 4
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "gsm_hard",
    "ok": true,
    "gpu": 4
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "arc_challenge",
    "ok": true,
    "gpu": 4
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "svamp",
    "ok": true,
    "gpu": 3
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "aqua_rat",
    "ok": true,
    "gpu": 3
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "humaneval",
    "ok": true,
    "gpu": 3
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "livecodebench_easy",
    "ok": true,
    "gpu": 3
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "medmcqa_easy",
    "ok": true,
    "gpu": 3
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "pubmedqa_pqal",
    "ok": true,
    "gpu": 3
  },
  {
    "side": "Y",
    "model": "meta-llama/Llama-3.2-3B-Instruct",
    "task": "mbpp_plus",
    "ok": true,
    "gpu": 3
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "svamp",
    "ok": true,
    "gpu": 2
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "aqua_rat",
    "ok": true,
    "gpu": 2
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "humaneval",
    "ok": true,
    "gpu": 2
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "livecodebench_easy",
    "ok": true,
    "gpu": 2
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "medmcqa_easy",
    "ok": true,
    "gpu": 2
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "pubmedqa_pqal",
    "ok": true,
    "gpu": 2
  },
  {
    "side": "X",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "mbpp_plus",
    "ok": true,
    "gpu": 2
  }
]