from __future__ import annotations import hashlib import json from dataclasses import dataclass from pathlib import Path from typing import Any MAX_SUBMISSION_BYTES = 100 * 1024 * 1024 @dataclass(frozen=True) class Track: name: str expected_samples: int TRACKS = { "test-500": Track("Official Test 500", 500), } REQUIRED_PREDICTION_FIELDS = { "index", "qid", "question", "planning_text", "api_input", "api_output", "result_edit", } def load_and_validate_submission(path: str | Path, track_id: str) -> tuple[list[dict[str, Any]], str]: submission_path = Path(path) if not submission_path.is_file(): raise ValueError("Submission file does not exist.") if submission_path.stat().st_size > MAX_SUBMISSION_BYTES: raise ValueError("Submission file exceeds the 100 MB limit.") if track_id not in TRACKS: raise ValueError(f"Unknown track: {track_id}") try: payload = json.loads(submission_path.read_text(encoding="utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise ValueError(f"Submission must be valid UTF-8 JSON: {exc}") from exc predictions = payload.get("predictions") if isinstance(payload, dict) else payload if not isinstance(predictions, list): raise ValueError('Submission must be a JSON list or {"predictions": [...]}.') expected_samples = TRACKS[track_id].expected_samples if len(predictions) != expected_samples: raise ValueError( f"Track {track_id} requires {expected_samples} predictions; received {len(predictions)}." ) seen_ids: set[str] = set() for position, prediction in enumerate(predictions, start=1): if not isinstance(prediction, dict): raise ValueError(f"Prediction #{position} must be a JSON object.") missing_fields = REQUIRED_PREDICTION_FIELDS - prediction.keys() if missing_fields: missing = ", ".join(sorted(missing_fields)) raise ValueError(f"Prediction #{position} is missing required fields: {missing}.") if "id" in prediction: raise ValueError(f"Prediction #{position} must use 'qid'; field 'id' is not accepted.") sequence_index = prediction["index"] if isinstance(sequence_index, bool) or not isinstance(sequence_index, int): raise ValueError(f"Prediction #{position} field 'index' must be an integer.") if sequence_index != position: raise ValueError( f"Prediction #{position} must have index {position}; received {sequence_index}." ) qid = prediction["qid"] if isinstance(qid, bool) or not isinstance(qid, (int, str)): raise ValueError(f"Prediction #{position} field 'qid' must be an integer or string.") prediction_id = str(qid).strip() if not prediction_id: raise ValueError(f"Prediction #{position} field 'qid' must not be empty.") if prediction_id in seen_ids: raise ValueError(f"Duplicate prediction qid: {prediction_id}") seen_ids.add(prediction_id) if not isinstance(prediction["question"], str): raise ValueError(f"Prediction {prediction_id} field 'question' must be a string.") if not isinstance(prediction["planning_text"], (str, list)): raise ValueError( f"Prediction {prediction_id} field 'planning_text' must be a string or list." ) if not isinstance(prediction["api_input"], (list, dict)): raise ValueError(f"Prediction {prediction_id} field 'api_input' must be a list or object.") if not isinstance(prediction["api_output"], (list, dict)): raise ValueError(f"Prediction {prediction_id} field 'api_output' must be a list or object.") if not isinstance(prediction["result_edit"], str): raise ValueError(f"Prediction {prediction_id} field 'result_edit' must be a string.") digest = hashlib.sha256(submission_path.read_bytes()).hexdigest() return predictions, digest