#!/usr/bin/env bash
set -euo pipefail
MODEL_PATH=${1:?Usage: eval/run_likelihood.sh MODEL_PATH_OR_REPO [OUTPUT_DIR]}
OUTPUT_DIR=${2:-eval_results}
mkdir -p "$OUTPUT_DIR"
accelerate launch -m eval.lm_eval --model dLLM \
  --model_args "model_path=${MODEL_PATH},likelihood_mc_num=32" \
  --tasks arc_easy,boolq,openbookqa,piqa,race,social_iqa \
  --num_fewshot 0 --batch_size 1 --output_path "$OUTPUT_DIR/likelihood.json"