[ { "id": "autobenchmarkcard:benchmark_details.name", "source": "autobenchmarkcard", "section": "benchmark_details", "field": "name", "schemaPath": "benchmark_details.name", "fullPath": "autobenchmarkcard.benchmark_details.name", "type": "str", "description": "Official name of the benchmark as it appears in literature", "required": "yes" }, { "id": "autobenchmarkcard:benchmark_details.overview", "source": "autobenchmarkcard", "section": "benchmark_details", "field": "overview", "schemaPath": "benchmark_details.overview", "fullPath": "autobenchmarkcard.benchmark_details.overview", "type": "str", "description": "Comprehensive 2-3 sentence description of what the benchmark measures, its characteristics, and significance", "required": "yes" }, { "id": "autobenchmarkcard:benchmark_details.data_type", "source": "autobenchmarkcard", "section": "benchmark_details", "field": "data_type", "schemaPath": "benchmark_details.data_type", "fullPath": "autobenchmarkcard.benchmark_details.data_type", "type": "str", "description": "Primary data modality (e.g. text, image, audio, multimodal, tabular)", "required": "yes" }, { "id": "autobenchmarkcard:benchmark_details.domains", "source": "autobenchmarkcard", "section": "benchmark_details", "field": "domains", "schemaPath": "benchmark_details.domains", "fullPath": "autobenchmarkcard.benchmark_details.domains", "type": "List[str]", "description": "Application domains or subject areas (e.g. medical, legal, scientific)", "required": "yes" }, { "id": "autobenchmarkcard:benchmark_details.languages", "source": "autobenchmarkcard", "section": "benchmark_details", "field": "languages", "schemaPath": "benchmark_details.languages", "fullPath": "autobenchmarkcard.benchmark_details.languages", "type": "List[str]", "description": "Languages supported, using full names (e.g. English, Chinese, Multilingual)", "required": "yes" }, { "id": "autobenchmarkcard:benchmark_details.similar_benchmarks", "source": "autobenchmarkcard", "section": "benchmark_details", "field": "similar_benchmarks", "schemaPath": "benchmark_details.similar_benchmarks", "fullPath": "autobenchmarkcard.benchmark_details.similar_benchmarks", "type": "List[str]", "description": "Names of closely related or comparable benchmarks", "required": "yes" }, { "id": "autobenchmarkcard:benchmark_details.resources", "source": "autobenchmarkcard", "section": "benchmark_details", "field": "resources", "schemaPath": "benchmark_details.resources", "fullPath": "autobenchmarkcard.benchmark_details.resources", "type": "List[str]", "description": "URLs to papers, datasets, leaderboards, and documentation", "required": "yes" }, { "id": "autobenchmarkcard:benchmark_details.provenance", "source": "autobenchmarkcard", "section": "benchmark_details", "field": "provenance", "schemaPath": "benchmark_details.provenance", "fullPath": "autobenchmarkcard.benchmark_details.provenance", "type": "dict", "description": "Maps each field to {source, evidence}. Present in all content sections.", "required": "no" }, { "id": "autobenchmarkcard:purpose_and_intended_users.goal", "source": "autobenchmarkcard", "section": "purpose_and_intended_users", "field": "goal", "schemaPath": "purpose_and_intended_users.goal", "fullPath": "autobenchmarkcard.purpose_and_intended_users.goal", "type": "str", "description": "Primary objective and research question the benchmark addresses", "required": "yes" }, { "id": "autobenchmarkcard:purpose_and_intended_users.audience", "source": "autobenchmarkcard", "section": "purpose_and_intended_users", "field": "audience", "schemaPath": "purpose_and_intended_users.audience", "fullPath": "autobenchmarkcard.purpose_and_intended_users.audience", "type": "List[str]", "description": "Target user groups (e.g. AI researchers, model developers, safety evaluators)", "required": "yes" }, { "id": "autobenchmarkcard:purpose_and_intended_users.tasks", "source": "autobenchmarkcard", "section": "purpose_and_intended_users", "field": "tasks", "schemaPath": "purpose_and_intended_users.tasks", "fullPath": "autobenchmarkcard.purpose_and_intended_users.tasks", "type": "List[str]", "description": "Specific evaluation tasks covered (e.g. question answering, code generation)", "required": "yes" }, { "id": "autobenchmarkcard:purpose_and_intended_users.limitations", "source": "autobenchmarkcard", "section": "purpose_and_intended_users", "field": "limitations", "schemaPath": "purpose_and_intended_users.limitations", "fullPath": "autobenchmarkcard.purpose_and_intended_users.limitations", "type": "str", "description": "Known limitations, biases, or constraints users should be aware of", "required": "yes" }, { "id": "autobenchmarkcard:purpose_and_intended_users.out_of_scope_uses", "source": "autobenchmarkcard", "section": "purpose_and_intended_users", "field": "out_of_scope_uses", "schemaPath": "purpose_and_intended_users.out_of_scope_uses", "fullPath": "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", "type": "List[str]", "description": "Explicit examples of inappropriate or unsupported use cases", "required": "yes" }, { "id": "autobenchmarkcard:purpose_and_intended_users.provenance", "source": "autobenchmarkcard", "section": "purpose_and_intended_users", "field": "provenance", "schemaPath": "purpose_and_intended_users.provenance", "fullPath": "autobenchmarkcard.purpose_and_intended_users.provenance", "type": "dict", "description": "Maps each field to {source, evidence}. Present in all content sections.", "required": "no" }, { "id": "autobenchmarkcard:data.source", "source": "autobenchmarkcard", "section": "data", "field": "source", "schemaPath": "data.source", "fullPath": "autobenchmarkcard.data.source", "type": "str", "description": "Data origins, collection methods, and any preprocessing steps applied", "required": "yes" }, { "id": "autobenchmarkcard:data.size", "source": "autobenchmarkcard", "section": "data", "field": "size", "schemaPath": "data.size", "fullPath": "autobenchmarkcard.data.size", "type": "str", "description": "Dataset size with specific numbers (e.g. 10,000 examples, 50K questions across 3 splits)", "required": "yes" }, { "id": "autobenchmarkcard:data.format", "source": "autobenchmarkcard", "section": "data", "field": "format", "schemaPath": "data.format", "fullPath": "autobenchmarkcard.data.format", "type": "str", "description": "Data structure and file formats (e.g. JSON with question-answer pairs)", "required": "yes" }, { "id": "autobenchmarkcard:data.annotation", "source": "autobenchmarkcard", "section": "data", "field": "annotation", "schemaPath": "data.annotation", "fullPath": "autobenchmarkcard.data.annotation", "type": "str", "description": "Annotation methodology, quality control, inter-annotator agreement, and human involvement", "required": "yes" }, { "id": "autobenchmarkcard:data.provenance", "source": "autobenchmarkcard", "section": "data", "field": "provenance", "schemaPath": "data.provenance", "fullPath": "autobenchmarkcard.data.provenance", "type": "dict", "description": "Maps each field to {source, evidence}. Present in all content sections.", "required": "no" }, { "id": "autobenchmarkcard:methodology.methods", "source": "autobenchmarkcard", "section": "methodology", "field": "methods", "schemaPath": "methodology.methods", "fullPath": "autobenchmarkcard.methodology.methods", "type": "List[str]", "description": "Evaluation approaches applied (e.g. zero-shot, few-shot prompting, fine-tuning)", "required": "yes" }, { "id": "autobenchmarkcard:methodology.metrics", "source": "autobenchmarkcard", "section": "methodology", "field": "metrics", "schemaPath": "methodology.metrics", "fullPath": "autobenchmarkcard.methodology.metrics", "type": "List[str]", "description": "Quantitative metrics used (e.g. accuracy, F1-score, BLEU, exact match)", "required": "yes" }, { "id": "autobenchmarkcard:methodology.calculation", "source": "autobenchmarkcard", "section": "methodology", "field": "calculation", "schemaPath": "methodology.calculation", "fullPath": "autobenchmarkcard.methodology.calculation", "type": "str", "description": "How metrics are computed, including normalization or aggregation methods", "required": "yes" }, { "id": "autobenchmarkcard:methodology.interpretation", "source": "autobenchmarkcard", "section": "methodology", "field": "interpretation", "schemaPath": "methodology.interpretation", "fullPath": "autobenchmarkcard.methodology.interpretation", "type": "str", "description": "Guidelines for interpreting scores, score ranges, and caveats", "required": "yes" }, { "id": "autobenchmarkcard:methodology.baseline_results", "source": "autobenchmarkcard", "section": "methodology", "field": "baseline_results", "schemaPath": "methodology.baseline_results", "fullPath": "autobenchmarkcard.methodology.baseline_results", "type": "str", "description": "Performance of established models or baselines with specific numbers", "required": "yes" }, { "id": "autobenchmarkcard:methodology.validation", "source": "autobenchmarkcard", "section": "methodology", "field": "validation", "schemaPath": "methodology.validation", "fullPath": "autobenchmarkcard.methodology.validation", "type": "str", "description": "Quality assurance measures and steps taken to ensure reproducible evaluations", "required": "yes" }, { "id": "autobenchmarkcard:methodology.provenance", "source": "autobenchmarkcard", "section": "methodology", "field": "provenance", "schemaPath": "methodology.provenance", "fullPath": "autobenchmarkcard.methodology.provenance", "type": "dict", "description": "Maps each field to {source, evidence}. Present in all content sections.", "required": "no" }, { "id": "autobenchmarkcard:ethical_and_legal_considerations.privacy_and_anonymity", "source": "autobenchmarkcard", "section": "ethical_and_legal_considerations", "field": "privacy_and_anonymity", "schemaPath": "ethical_and_legal_considerations.privacy_and_anonymity", "fullPath": "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", "type": "str", "description": "Data protection measures, anonymization techniques, and handling of PII", "required": "yes" }, { "id": "autobenchmarkcard:ethical_and_legal_considerations.data_licensing", "source": "autobenchmarkcard", "section": "ethical_and_legal_considerations", "field": "data_licensing", "schemaPath": "ethical_and_legal_considerations.data_licensing", "fullPath": "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", "type": "str", "description": "License terms, usage restrictions, and redistribution permissions", "required": "yes" }, { "id": "autobenchmarkcard:ethical_and_legal_considerations.consent_procedures", "source": "autobenchmarkcard", "section": "ethical_and_legal_considerations", "field": "consent_procedures", "schemaPath": "ethical_and_legal_considerations.consent_procedures", "fullPath": "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", "type": "str", "description": "Informed consent processes, participant rights, and withdrawal procedures", "required": "yes" }, { "id": "autobenchmarkcard:ethical_and_legal_considerations.compliance_with_regulations", "source": "autobenchmarkcard", "section": "ethical_and_legal_considerations", "field": "compliance_with_regulations", "schemaPath": "ethical_and_legal_considerations.compliance_with_regulations", "fullPath": "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", "type": "str", "description": "Adherence to GDPR, IRB approval, and other ethical review processes", "required": "yes" }, { "id": "autobenchmarkcard:ethical_and_legal_considerations.provenance", "source": "autobenchmarkcard", "section": "ethical_and_legal_considerations", "field": "provenance", "schemaPath": "ethical_and_legal_considerations.provenance", "fullPath": "autobenchmarkcard.ethical_and_legal_considerations.provenance", "type": "dict", "description": "Maps each field to {source, evidence}. Present in all content sections.", "required": "no" }, { "id": "autobenchmarkcard:possible_risks", "source": "autobenchmarkcard", "section": "possible_risks", "field": "", "schemaPath": "possible_risks", "fullPath": "autobenchmarkcard.possible_risks", "type": "List[object]", "description": "IBM Risk Atlas categories matched to this benchmark. Added by the risk worker.", "required": "no" }, { "id": "autobenchmarkcard:possible_risks.category", "source": "autobenchmarkcard", "section": "possible_risks", "field": "category", "schemaPath": "possible_risks.category", "fullPath": "autobenchmarkcard.possible_risks.category", "type": "str", "description": "Risk category name", "required": "no" }, { "id": "autobenchmarkcard:possible_risks.description", "source": "autobenchmarkcard", "section": "possible_risks", "field": "description", "schemaPath": "possible_risks.description", "fullPath": "autobenchmarkcard.possible_risks.description", "type": "str", "description": "Description of the risk", "required": "no" }, { "id": "autobenchmarkcard:possible_risks.type", "source": "autobenchmarkcard", "section": "possible_risks", "field": "type", "schemaPath": "possible_risks.type", "fullPath": "autobenchmarkcard.possible_risks.type", "type": "str", "description": "Risk type classification", "required": "no" }, { "id": "autobenchmarkcard:possible_risks.concern", "source": "autobenchmarkcard", "section": "possible_risks", "field": "concern", "schemaPath": "possible_risks.concern", "fullPath": "autobenchmarkcard.possible_risks.concern", "type": "str", "description": "Specific concern this risk raises", "required": "no" }, { "id": "autobenchmarkcard:possible_risks.url", "source": "autobenchmarkcard", "section": "possible_risks", "field": "url", "schemaPath": "possible_risks.url", "fullPath": "autobenchmarkcard.possible_risks.url", "type": "str | null", "description": "Link to the risk entry in IBM Risk Atlas", "required": "no" }, { "id": "autobenchmarkcard:possible_risks.taxonomy", "source": "autobenchmarkcard", "section": "possible_risks", "field": "taxonomy", "schemaPath": "possible_risks.taxonomy", "fullPath": "autobenchmarkcard.possible_risks.taxonomy", "type": "str", "description": "Taxonomy this risk belongs to", "required": "no" }, { "id": "autobenchmarkcard:flagged_fields.
.", "source": "autobenchmarkcard", "section": "flagged_fields", "field": "
.", "schemaPath": "flagged_fields.
.", "fullPath": "autobenchmarkcard.flagged_fields.
.", "type": "str", "description": "Added by factreasoner worker. Omitted if all fields pass. Value is one of: '[Possible Hallucination], no supporting evidence found in source material' or '[Factuality Score: X.XX], low factual alignment with source material'", "required": "no" }, { "id": "eee_eval:schema_version", "source": "eee_eval", "section": "schema_version", "field": "", "schemaPath": "schema_version", "fullPath": "eee_eval.schema_version", "type": "string", "description": "Version of the schema used for this evaluation data", "required": "yes" }, { "id": "eee_eval:evaluation_id", "source": "eee_eval", "section": "evaluation_id", "field": "", "schemaPath": "evaluation_id", "fullPath": "eee_eval.evaluation_id", "type": "string", "description": "Unique identifier for this specific evaluation run. Use eval_name/model_id/retrieved_timestamp format", "required": "yes" }, { "id": "eee_eval:evaluation_timestamp", "source": "eee_eval", "section": "evaluation_timestamp", "field": "", "schemaPath": "evaluation_timestamp", "fullPath": "eee_eval.evaluation_timestamp", "type": "string", "description": "Timestamp for when the evaluation was run", "required": "no" }, { "id": "eee_eval:retrieved_timestamp", "source": "eee_eval", "section": "retrieved_timestamp", "field": "", "schemaPath": "retrieved_timestamp", "fullPath": "eee_eval.retrieved_timestamp", "type": "string", "description": "Timestamp for when this record was created - using Unix Epoch time format", "required": "yes" }, { "id": "eee_eval:source_metadata", "source": "eee_eval", "section": "source_metadata", "field": "", "schemaPath": "source_metadata", "fullPath": "eee_eval.source_metadata", "type": "object", "description": "Metadata about the source of the leaderboard data", "required": "yes" }, { "id": "eee_eval:source_metadata.source_name", "source": "eee_eval", "section": "source_metadata", "field": "source_name", "schemaPath": "source_metadata.source_name", "fullPath": "eee_eval.source_metadata.source_name", "type": "string", "description": "Name of the source (e.g. title of the source leaderboard or name of the platform used for the evaluation).", "required": "no" }, { "id": "eee_eval:source_metadata.source_type", "source": "eee_eval", "section": "source_metadata", "field": "source_type", "schemaPath": "source_metadata.source_type", "fullPath": "eee_eval.source_metadata.source_type", "type": "enum: \"documentation\" | \"evaluation_run\"", "description": "Whether the data comes from a direct evaluation run or from documentation", "required": "yes" }, { "id": "eee_eval:source_metadata.source_organization_name", "source": "eee_eval", "section": "source_metadata", "field": "source_organization_name", "schemaPath": "source_metadata.source_organization_name", "fullPath": "eee_eval.source_metadata.source_organization_name", "type": "string", "description": "Name of the organization that provides the data", "required": "yes" }, { "id": "eee_eval:source_metadata.source_organization_url", "source": "eee_eval", "section": "source_metadata", "field": "source_organization_url", "schemaPath": "source_metadata.source_organization_url", "fullPath": "eee_eval.source_metadata.source_organization_url", "type": "string", "description": "URL for the organization that provides the data", "required": "no" }, { "id": "eee_eval:source_metadata.source_organization_logo_url", "source": "eee_eval", "section": "source_metadata", "field": "source_organization_logo_url", "schemaPath": "source_metadata.source_organization_logo_url", "fullPath": "eee_eval.source_metadata.source_organization_logo_url", "type": "string", "description": "URL for the Logo for the organization that provides the data", "required": "no" }, { "id": "eee_eval:source_metadata.evaluator_relationship", "source": "eee_eval", "section": "source_metadata", "field": "evaluator_relationship", "schemaPath": "source_metadata.evaluator_relationship", "fullPath": "eee_eval.source_metadata.evaluator_relationship", "type": "enum: \"first_party\" | \"third_party\" | \"collaborative\" | \"other\"", "description": "Relationship between the evaluator and the model", "required": "yes" }, { "id": "eee_eval:source_metadata.additional_details", "source": "eee_eval", "section": "source_metadata", "field": "additional_details", "schemaPath": "source_metadata.additional_details", "fullPath": "eee_eval.source_metadata.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:eval_library", "source": "eee_eval", "section": "eval_library", "field": "", "schemaPath": "eval_library", "fullPath": "eee_eval.eval_library", "type": "object", "description": "Evaluation library/framework used to run the evaluation", "required": "yes" }, { "id": "eee_eval:eval_library.name", "source": "eee_eval", "section": "eval_library", "field": "name", "schemaPath": "eval_library.name", "fullPath": "eee_eval.eval_library.name", "type": "string", "description": "Name of the evaluation library (e.g. inspect_ai, lm_eval, helm)", "required": "yes" }, { "id": "eee_eval:eval_library.version", "source": "eee_eval", "section": "eval_library", "field": "version", "schemaPath": "eval_library.version", "fullPath": "eee_eval.eval_library.version", "type": "string", "description": "Version of the evaluation library. Use 'unknown' if the version is not available.", "required": "yes" }, { "id": "eee_eval:eval_library.additional_details", "source": "eee_eval", "section": "eval_library", "field": "additional_details", "schemaPath": "eval_library.additional_details", "fullPath": "eee_eval.eval_library.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:model_info", "source": "eee_eval", "section": "model_info", "field": "", "schemaPath": "model_info", "fullPath": "eee_eval.model_info", "type": "object", "description": "Complete model specification including basic information, technical configuration and inference settings", "required": "yes" }, { "id": "eee_eval:model_info.name", "source": "eee_eval", "section": "model_info", "field": "name", "schemaPath": "model_info.name", "fullPath": "eee_eval.model_info.name", "type": "string", "description": "Model name provided by evaluation source", "required": "yes" }, { "id": "eee_eval:model_info.id", "source": "eee_eval", "section": "model_info", "field": "id", "schemaPath": "model_info.id", "fullPath": "eee_eval.model_info.id", "type": "string", "description": "Model name in HuggingFace format (e.g. meta-llama/Llama-3.1-8B-Instruct for models available on HuggingFace or openai/azure/gpt-4o-mini-2024-07-18 for closed API models)", "required": "yes" }, { "id": "eee_eval:model_info.developer", "source": "eee_eval", "section": "model_info", "field": "developer", "schemaPath": "model_info.developer", "fullPath": "eee_eval.model_info.developer", "type": "string", "description": "Name of organization that provides the model (e.g. 'OpenAI')", "required": "no" }, { "id": "eee_eval:model_info.inference_platform", "source": "eee_eval", "section": "model_info", "field": "inference_platform", "schemaPath": "model_info.inference_platform", "fullPath": "eee_eval.model_info.inference_platform", "type": "string", "description": "Name of inference platform which provides an access to models by API to run the evaluations or provides models weights to run them locally (e.g. HuggingFace, Bedrock, Together AI)", "required": "no" }, { "id": "eee_eval:model_info.inference_engine", "source": "eee_eval", "section": "model_info", "field": "inference_engine", "schemaPath": "model_info.inference_engine", "fullPath": "eee_eval.model_info.inference_engine", "type": "object", "description": "Name of inference engine which provides an access to optimized models to use them for local evaluations (e.g. vLLM, Ollama).", "required": "no" }, { "id": "eee_eval:model_info.inference_engine.name", "source": "eee_eval", "section": "model_info", "field": "inference_engine.name", "schemaPath": "model_info.inference_engine.name", "fullPath": "eee_eval.model_info.inference_engine.name", "type": "string", "description": "Name of the inference engine", "required": "no" }, { "id": "eee_eval:model_info.inference_engine.version", "source": "eee_eval", "section": "model_info", "field": "inference_engine.version", "schemaPath": "model_info.inference_engine.version", "fullPath": "eee_eval.model_info.inference_engine.version", "type": "string", "description": "Version of the inference engine", "required": "no" }, { "id": "eee_eval:model_info.additional_details", "source": "eee_eval", "section": "model_info", "field": "additional_details", "schemaPath": "model_info.additional_details", "fullPath": "eee_eval.model_info.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results", "source": "eee_eval", "section": "evaluation_results", "field": "", "schemaPath": "evaluation_results", "fullPath": "eee_eval.evaluation_results", "type": "array[object]", "description": "Array of evaluation results", "required": "yes" }, { "id": "eee_eval:evaluation_results.evaluation_result_id", "source": "eee_eval", "section": "evaluation_results", "field": "evaluation_result_id", "schemaPath": "evaluation_results.evaluation_result_id", "fullPath": "eee_eval.evaluation_results.evaluation_result_id", "type": "string", "description": "Stable identifier for this metric result inside an evaluation run. Recommended deterministic join key for instance-level records.", "required": "no" }, { "id": "eee_eval:evaluation_results.evaluation_name", "source": "eee_eval", "section": "evaluation_results", "field": "evaluation_name", "schemaPath": "evaluation_results.evaluation_name", "fullPath": "eee_eval.evaluation_results.evaluation_name", "type": "string", "description": "Name of the evaluation", "required": "yes" }, { "id": "eee_eval:evaluation_results.source_data", "source": "eee_eval", "section": "evaluation_results", "field": "source_data", "schemaPath": "evaluation_results.source_data", "fullPath": "eee_eval.evaluation_results.source_data", "type": "oneOf: url | hf_dataset | other", "description": "Source of dataset for this evaluation: URL, HuggingFace dataset, or private/custom dataset.", "required": "yes" }, { "id": "eee_eval:evaluation_results.source_data.dataset_name", "source": "eee_eval", "section": "evaluation_results", "field": "source_data.dataset_name", "schemaPath": "evaluation_results.source_data.dataset_name", "fullPath": "eee_eval.evaluation_results.source_data.dataset_name", "type": "string", "description": "Name of the source dataset", "required": "yes" }, { "id": "eee_eval:evaluation_results.source_data.url", "source": "eee_eval", "section": "evaluation_results", "field": "source_data.url", "schemaPath": "evaluation_results.source_data.url", "fullPath": "eee_eval.evaluation_results.source_data.url", "type": "array[string]", "description": "URL(s) for the source of the evaluation data [only when source_type=url]", "required": "yes" }, { "id": "eee_eval:evaluation_results.source_data.additional_details", "source": "eee_eval", "section": "evaluation_results", "field": "source_data.additional_details", "schemaPath": "evaluation_results.source_data.additional_details", "fullPath": "eee_eval.evaluation_results.source_data.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results.source_data.hf_repo", "source": "eee_eval", "section": "evaluation_results", "field": "source_data.hf_repo", "schemaPath": "evaluation_results.source_data.hf_repo", "fullPath": "eee_eval.evaluation_results.source_data.hf_repo", "type": "string", "description": "HuggingFace repository identifier [only when source_type=hf_dataset]", "required": "no" }, { "id": "eee_eval:evaluation_results.source_data.hf_split", "source": "eee_eval", "section": "evaluation_results", "field": "source_data.hf_split", "schemaPath": "evaluation_results.source_data.hf_split", "fullPath": "eee_eval.evaluation_results.source_data.hf_split", "type": "string", "description": "One of train, val or test. [only when source_type=hf_dataset]", "required": "no" }, { "id": "eee_eval:evaluation_results.source_data.samples_number", "source": "eee_eval", "section": "evaluation_results", "field": "source_data.samples_number", "schemaPath": "evaluation_results.source_data.samples_number", "fullPath": "eee_eval.evaluation_results.source_data.samples_number", "type": "integer", "description": "Number of samples in the dataset [only when source_type=hf_dataset]", "required": "no" }, { "id": "eee_eval:evaluation_results.source_data.sample_ids", "source": "eee_eval", "section": "evaluation_results", "field": "source_data.sample_ids", "schemaPath": "evaluation_results.source_data.sample_ids", "fullPath": "eee_eval.evaluation_results.source_data.sample_ids", "type": "array[string]", "description": "Array of sample ids used for evaluation [only when source_type=hf_dataset]", "required": "no" }, { "id": "eee_eval:evaluation_results.evaluation_timestamp", "source": "eee_eval", "section": "evaluation_results", "field": "evaluation_timestamp", "schemaPath": "evaluation_results.evaluation_timestamp", "fullPath": "eee_eval.evaluation_results.evaluation_timestamp", "type": "string", "description": "Timestamp for when the evaluations were run", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config", "schemaPath": "evaluation_results.metric_config", "fullPath": "eee_eval.evaluation_results.metric_config", "type": "object", "description": "Details about the metric", "required": "yes" }, { "id": "eee_eval:evaluation_results.metric_config.evaluation_description", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.evaluation_description", "schemaPath": "evaluation_results.metric_config.evaluation_description", "fullPath": "eee_eval.evaluation_results.metric_config.evaluation_description", "type": "string", "description": "Description of the evaluation", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.metric_id", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.metric_id", "schemaPath": "evaluation_results.metric_config.metric_id", "fullPath": "eee_eval.evaluation_results.metric_config.metric_id", "type": "string", "description": "Stable metric identifier for joining/deduping/querying. Use a canonical global id when applicable (e.g. accuracy, f1_macro, auroc, rmse, pass_at_k). For benchmark/leaderboard-specific metrics, use a namespaced id (e.g. rewardbench.overall, lmarena.elo).", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.metric_name", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.metric_name", "schemaPath": "evaluation_results.metric_config.metric_name", "fullPath": "eee_eval.evaluation_results.metric_config.metric_name", "type": "string", "description": "Display name for the metric (e.g. Accuracy, F1-macro, pass@1).", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.metric_kind", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.metric_kind", "schemaPath": "evaluation_results.metric_config.metric_kind", "fullPath": "eee_eval.evaluation_results.metric_config.metric_kind", "type": "string", "description": "Normalized metric family/type used for safe aggregation (e.g. accuracy, f1, auroc, rmse, mae, pass_rate, elo).", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.metric_unit", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.metric_unit", "schemaPath": "evaluation_results.metric_config.metric_unit", "fullPath": "eee_eval.evaluation_results.metric_config.metric_unit", "type": "string", "description": "Unit of the metric values (e.g. proportion, percent, points, ms, tokens).", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.metric_parameters", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.metric_parameters", "schemaPath": "evaluation_results.metric_config.metric_parameters", "fullPath": "eee_eval.evaluation_results.metric_config.metric_parameters", "type": "dict[str, string | number | boolean | null]", "description": "Metric-specific parameters (e.g. {\"k\": 1} for pass@k).", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.lower_is_better", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.lower_is_better", "schemaPath": "evaluation_results.metric_config.lower_is_better", "fullPath": "eee_eval.evaluation_results.metric_config.lower_is_better", "type": "boolean", "description": "Whether a lower score is better", "required": "yes" }, { "id": "eee_eval:evaluation_results.metric_config.score_type", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.score_type", "schemaPath": "evaluation_results.metric_config.score_type", "fullPath": "eee_eval.evaluation_results.metric_config.score_type", "type": "enum: \"binary\" | \"continuous\" | \"levels\"", "description": "Type of score", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.level_names", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.level_names", "schemaPath": "evaluation_results.metric_config.level_names", "fullPath": "eee_eval.evaluation_results.metric_config.level_names", "type": "array[string]", "description": "Names of the score levels", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.level_metadata", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.level_metadata", "schemaPath": "evaluation_results.metric_config.level_metadata", "fullPath": "eee_eval.evaluation_results.metric_config.level_metadata", "type": "array[string]", "description": "Additional Description for each Score Level", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.has_unknown_level", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.has_unknown_level", "schemaPath": "evaluation_results.metric_config.has_unknown_level", "fullPath": "eee_eval.evaluation_results.metric_config.has_unknown_level", "type": "boolean", "description": "Indicates whether there is an Unknown Level - if True, then a score of -1 will be treated as Unknown", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.min_score", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.min_score", "schemaPath": "evaluation_results.metric_config.min_score", "fullPath": "eee_eval.evaluation_results.metric_config.min_score", "type": "number", "description": "Minimum possible score for continuous metric", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.max_score", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.max_score", "schemaPath": "evaluation_results.metric_config.max_score", "fullPath": "eee_eval.evaluation_results.metric_config.max_score", "type": "number", "description": "Maximum possible score for continuous metric", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring", "schemaPath": "evaluation_results.metric_config.llm_scoring", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring", "type": "object", "description": "Configuration when LLM is used as scorer/judge", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges", "type": "array[object]", "description": "LLM judge(s) - single item for judge, multiple for jury", "required": "yes" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].model_info", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].model_info", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].model_info", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].model_info", "type": "object", "description": "Complete model specification including basic information, technical configuration and inference settings", "required": "yes" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].model_info.name", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].model_info.name", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].model_info.name", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].model_info.name", "type": "string", "description": "Model name provided by evaluation source", "required": "yes" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].model_info.id", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].model_info.id", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].model_info.id", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].model_info.id", "type": "string", "description": "Model name in HuggingFace format (e.g. meta-llama/Llama-3.1-8B-Instruct for models available on HuggingFace or openai/azure/gpt-4o-mini-2024-07-18 for closed API models)", "required": "yes" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].model_info.developer", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].model_info.developer", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].model_info.developer", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].model_info.developer", "type": "string", "description": "Name of organization that provides the model (e.g. 'OpenAI')", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_platform", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].model_info.inference_platform", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_platform", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_platform", "type": "string", "description": "Name of inference platform which provides an access to models by API to run the evaluations or provides models weights to run them locally (e.g. HuggingFace, Bedrock, Together AI)", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_engine", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].model_info.inference_engine", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_engine", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_engine", "type": "object", "description": "Name of inference engine which provides an access to optimized models to use them for local evaluations (e.g. vLLM, Ollama).", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_engine.name", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].model_info.inference_engine.name", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_engine.name", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_engine.name", "type": "string", "description": "Name of the inference engine", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_engine.version", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].model_info.inference_engine.version", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_engine.version", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].model_info.inference_engine.version", "type": "string", "description": "Version of the inference engine", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].model_info.additional_details", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].model_info.additional_details", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].model_info.additional_details", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].model_info.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].temperature", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].temperature", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].temperature", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].temperature", "type": "number", "description": "Sampling temperature for this judge", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].weight", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].weight", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].weight", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].weight", "type": "number", "description": "Weight of this judge's score in aggregation (used in jury)", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.judges[].additional_details", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.judges[].additional_details", "schemaPath": "evaluation_results.metric_config.llm_scoring.judges[].additional_details", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.judges[].additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.input_prompt", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.input_prompt", "schemaPath": "evaluation_results.metric_config.llm_scoring.input_prompt", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.input_prompt", "type": "string", "description": "Prompt template used for judging", "required": "yes" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.aggregation_method", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.aggregation_method", "schemaPath": "evaluation_results.metric_config.llm_scoring.aggregation_method", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.aggregation_method", "type": "enum: \"majority_vote\" | \"average\" | \"weighted_average\" | \"median\"", "description": "How to aggregate scores when multiple judges", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.expert_baseline", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.expert_baseline", "schemaPath": "evaluation_results.metric_config.llm_scoring.expert_baseline", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.expert_baseline", "type": "number", "description": "Expert/human baseline score for comparison", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.llm_scoring.additional_details", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.llm_scoring.additional_details", "schemaPath": "evaluation_results.metric_config.llm_scoring.additional_details", "fullPath": "eee_eval.evaluation_results.metric_config.llm_scoring.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results.metric_config.additional_details", "source": "eee_eval", "section": "evaluation_results", "field": "metric_config.additional_details", "schemaPath": "evaluation_results.metric_config.additional_details", "fullPath": "eee_eval.evaluation_results.metric_config.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details", "source": "eee_eval", "section": "evaluation_results", "field": "score_details", "schemaPath": "evaluation_results.score_details", "fullPath": "eee_eval.evaluation_results.score_details", "type": "object", "description": "The score for the evaluation and related details", "required": "yes" }, { "id": "eee_eval:evaluation_results.score_details.score", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.score", "schemaPath": "evaluation_results.score_details.score", "fullPath": "eee_eval.evaluation_results.score_details.score", "type": "number", "description": "The score for the evaluation", "required": "yes" }, { "id": "eee_eval:evaluation_results.score_details.details", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.details", "schemaPath": "evaluation_results.score_details.details", "fullPath": "eee_eval.evaluation_results.score_details.details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty", "schemaPath": "evaluation_results.score_details.uncertainty", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty", "type": "object", "description": "Quantification of uncertainty around the reported score", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.standard_error", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.standard_error", "schemaPath": "evaluation_results.score_details.uncertainty.standard_error", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.standard_error", "type": "object", "description": "Standard error of the score estimate (SE_mean = standard_deviation / sqrt(num_samples))", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.standard_error.value", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.standard_error.value", "schemaPath": "evaluation_results.score_details.uncertainty.standard_error.value", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.standard_error.value", "type": "number", "description": "The standard error value", "required": "yes" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.standard_error.method", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.standard_error.method", "schemaPath": "evaluation_results.score_details.uncertainty.standard_error.method", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.standard_error.method", "type": "string", "description": "How the standard error was computed (e.g. 'analytic', 'bootstrap', 'jackknife')", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.confidence_interval", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.confidence_interval", "schemaPath": "evaluation_results.score_details.uncertainty.confidence_interval", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.confidence_interval", "type": "object", "description": "Lower and upper bounds for the metric at a given confidence level.", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.confidence_interval.lower", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.confidence_interval.lower", "schemaPath": "evaluation_results.score_details.uncertainty.confidence_interval.lower", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.confidence_interval.lower", "type": "number", "description": "Lower bound of the confidence interval", "required": "yes" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.confidence_interval.upper", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.confidence_interval.upper", "schemaPath": "evaluation_results.score_details.uncertainty.confidence_interval.upper", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.confidence_interval.upper", "type": "number", "description": "Upper bound of the confidence interval", "required": "yes" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.confidence_interval.confidence_level", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.confidence_interval.confidence_level", "schemaPath": "evaluation_results.score_details.uncertainty.confidence_interval.confidence_level", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.confidence_interval.confidence_level", "type": "number", "description": "Confidence level (e.g. 0.95 for a 95% confidence interval)", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.confidence_interval.method", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.confidence_interval.method", "schemaPath": "evaluation_results.score_details.uncertainty.confidence_interval.method", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.confidence_interval.method", "type": "string", "description": "How the confidence interval was computed", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.standard_deviation", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.standard_deviation", "schemaPath": "evaluation_results.score_details.uncertainty.standard_deviation", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.standard_deviation", "type": "number", "description": "Standard deviation of the per-sample scores", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.num_samples", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.num_samples", "schemaPath": "evaluation_results.score_details.uncertainty.num_samples", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.num_samples", "type": "integer", "description": "Number of samples used to compute the uncertainty estimates", "required": "no" }, { "id": "eee_eval:evaluation_results.score_details.uncertainty.num_bootstrap_samples", "source": "eee_eval", "section": "evaluation_results", "field": "score_details.uncertainty.num_bootstrap_samples", "schemaPath": "evaluation_results.score_details.uncertainty.num_bootstrap_samples", "fullPath": "eee_eval.evaluation_results.score_details.uncertainty.num_bootstrap_samples", "type": "integer", "description": "Number of bootstrap resamples used, if bootstrap method was applied", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config", "schemaPath": "evaluation_results.generation_config", "fullPath": "eee_eval.evaluation_results.generation_config", "type": "object", "description": "", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args", "schemaPath": "evaluation_results.generation_config.generation_args", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args", "type": "object", "description": "Parameters used to generate results - properties may vary by model type", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.temperature", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.temperature", "schemaPath": "evaluation_results.generation_config.generation_args.temperature", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.temperature", "type": "null | number", "description": "Sampling temperature", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.top_p", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.top_p", "schemaPath": "evaluation_results.generation_config.generation_args.top_p", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.top_p", "type": "null | number", "description": "Nucleus sampling parameter", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.top_k", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.top_k", "schemaPath": "evaluation_results.generation_config.generation_args.top_k", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.top_k", "type": "null | number", "description": "Top-k sampling parameter", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.max_tokens", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.max_tokens", "schemaPath": "evaluation_results.generation_config.generation_args.max_tokens", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.max_tokens", "type": "integer", "description": "Maximum number of tokens to generate", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.execution_command", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.execution_command", "schemaPath": "evaluation_results.generation_config.generation_args.execution_command", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.execution_command", "type": "string", "description": "Command used to run the model to generate results", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.reasoning", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.reasoning", "schemaPath": "evaluation_results.generation_config.generation_args.reasoning", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.reasoning", "type": "boolean", "description": "Whether reasoning orchain-of-thought was used to generate results", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.prompt_template", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.prompt_template", "schemaPath": "evaluation_results.generation_config.generation_args.prompt_template", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.prompt_template", "type": "string", "description": "Input prompt template for task (should contain agentic info if needed).", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.agentic_eval_config", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.agentic_eval_config", "schemaPath": "evaluation_results.generation_config.generation_args.agentic_eval_config", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.agentic_eval_config", "type": "object", "description": "General configuration for agentic evaluations.", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.agentic_eval_config.available_tools", "schemaPath": "evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools", "type": "array[object]", "description": "List of all available tools with their configurations", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools[].name", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.agentic_eval_config.available_tools[].name", "schemaPath": "evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools[].name", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools[].name", "type": "string", "description": "e.g. bash, calculator, ...", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools[].description", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.agentic_eval_config.available_tools[].description", "schemaPath": "evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools[].description", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools[].description", "type": "string", "description": "", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools[].parameters", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.agentic_eval_config.available_tools[].parameters", "schemaPath": "evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools[].parameters", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.agentic_eval_config.available_tools[].parameters", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.agentic_eval_config.additional_details", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.agentic_eval_config.additional_details", "schemaPath": "evaluation_results.generation_config.generation_args.agentic_eval_config.additional_details", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.agentic_eval_config.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.eval_plan", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.eval_plan", "schemaPath": "evaluation_results.generation_config.generation_args.eval_plan", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.eval_plan", "type": "object", "description": "Plan (solvers) used in evaluation. Solvers are crucial parts of Inspect evaluations which can serve a wide variety of purposes like providing system prompts, prompt engineering, model generation or multi-turn dialog.", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.eval_plan.name ", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.eval_plan.name ", "schemaPath": "evaluation_results.generation_config.generation_args.eval_plan.name ", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.eval_plan.name ", "type": "string", "description": "", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.eval_plan.steps", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.eval_plan.steps", "schemaPath": "evaluation_results.generation_config.generation_args.eval_plan.steps", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.eval_plan.steps", "type": "array[string]", "description": "Array of evaluation plan steps (each step is a JSON-serialized string)", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.eval_plan.config", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.eval_plan.config", "schemaPath": "evaluation_results.generation_config.generation_args.eval_plan.config", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.eval_plan.config", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.eval_limits", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.eval_limits", "schemaPath": "evaluation_results.generation_config.generation_args.eval_limits", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.eval_limits", "type": "object", "description": "Listed evaluation limits like time limit, message limit, token limit.", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.eval_limits.time_limit", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.eval_limits.time_limit", "schemaPath": "evaluation_results.generation_config.generation_args.eval_limits.time_limit", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.eval_limits.time_limit", "type": "integer", "description": "Time limit for evaluation.", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.eval_limits.message_limit", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.eval_limits.message_limit", "schemaPath": "evaluation_results.generation_config.generation_args.eval_limits.message_limit", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.eval_limits.message_limit", "type": "integer", "description": "Message limit for evaluation.", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.eval_limits.token_limit", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.eval_limits.token_limit", "schemaPath": "evaluation_results.generation_config.generation_args.eval_limits.token_limit", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.eval_limits.token_limit", "type": "integer", "description": "Token limit for evaluation.", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.sandbox", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.sandbox", "schemaPath": "evaluation_results.generation_config.generation_args.sandbox", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.sandbox", "type": "object", "description": "", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.sandbox.type", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.sandbox.type", "schemaPath": "evaluation_results.generation_config.generation_args.sandbox.type", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.sandbox.type", "type": "string", "description": "Type of sandbox e.g. docker", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.sandbox.config", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.sandbox.config", "schemaPath": "evaluation_results.generation_config.generation_args.sandbox.config", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.sandbox.config", "type": "string", "description": "Config file name/path e.g. compose.yaml. TODO or full config? Not sure based on the Inspect docs", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.max_attempts", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.max_attempts", "schemaPath": "evaluation_results.generation_config.generation_args.max_attempts", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.max_attempts", "type": "integer", "description": "Maximum number of submission attempts (default 1).", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.generation_args.incorrect_attempt_feedback", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.generation_args.incorrect_attempt_feedback", "schemaPath": "evaluation_results.generation_config.generation_args.incorrect_attempt_feedback", "fullPath": "eee_eval.evaluation_results.generation_config.generation_args.incorrect_attempt_feedback", "type": "string", "description": "Feedback from the model after incorrect attempt.", "required": "no" }, { "id": "eee_eval:evaluation_results.generation_config.additional_details", "source": "eee_eval", "section": "evaluation_results", "field": "generation_config.additional_details", "schemaPath": "evaluation_results.generation_config.additional_details", "fullPath": "eee_eval.evaluation_results.generation_config.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_eval:detailed_evaluation_results", "source": "eee_eval", "section": "detailed_evaluation_results", "field": "", "schemaPath": "detailed_evaluation_results", "fullPath": "eee_eval.detailed_evaluation_results", "type": "object", "description": "Reference to the evaluation results for all individual samples in the evaluation", "required": "no" }, { "id": "eee_eval:detailed_evaluation_results.format", "source": "eee_eval", "section": "detailed_evaluation_results", "field": "format", "schemaPath": "detailed_evaluation_results.format", "fullPath": "eee_eval.detailed_evaluation_results.format", "type": "enum: \"jsonl\" | \"json\"", "description": "Format of the detailed evaluation results", "required": "no" }, { "id": "eee_eval:detailed_evaluation_results.file_path", "source": "eee_eval", "section": "detailed_evaluation_results", "field": "file_path", "schemaPath": "detailed_evaluation_results.file_path", "fullPath": "eee_eval.detailed_evaluation_results.file_path", "type": "string", "description": "Path to the detailed evaluation results file", "required": "no" }, { "id": "eee_eval:detailed_evaluation_results.hash_algorithm", "source": "eee_eval", "section": "detailed_evaluation_results", "field": "hash_algorithm", "schemaPath": "detailed_evaluation_results.hash_algorithm", "fullPath": "eee_eval.detailed_evaluation_results.hash_algorithm", "type": "enum: \"sha256\" | \"md5\"", "description": "Hash algorithm used for checksum and sample_hash in instance-level data", "required": "no" }, { "id": "eee_eval:detailed_evaluation_results.checksum", "source": "eee_eval", "section": "detailed_evaluation_results", "field": "checksum", "schemaPath": "detailed_evaluation_results.checksum", "fullPath": "eee_eval.detailed_evaluation_results.checksum", "type": "string", "description": "Checksum value of the file", "required": "no" }, { "id": "eee_eval:detailed_evaluation_results.total_rows", "source": "eee_eval", "section": "detailed_evaluation_results", "field": "total_rows", "schemaPath": "detailed_evaluation_results.total_rows", "fullPath": "eee_eval.detailed_evaluation_results.total_rows", "type": "integer", "description": "Total number of rows in the detailed evaluation results file", "required": "no" }, { "id": "eee_eval:detailed_evaluation_results.additional_details", "source": "eee_eval", "section": "detailed_evaluation_results", "field": "additional_details", "schemaPath": "detailed_evaluation_results.additional_details", "fullPath": "eee_eval.detailed_evaluation_results.additional_details", "type": "dict[str, string]", "description": "Additional parameters (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_instance_level_eval:schema_version", "source": "eee_instance_level_eval", "section": "schema_version", "field": "", "schemaPath": "schema_version", "fullPath": "eee_instance_level_eval.schema_version", "type": "string", "description": "Version of the schema used for this instance data", "required": "yes" }, { "id": "eee_instance_level_eval:evaluation_id", "source": "eee_instance_level_eval", "section": "evaluation_id", "field": "", "schemaPath": "evaluation_id", "fullPath": "eee_instance_level_eval.evaluation_id", "type": "string", "description": "Foreign key linking to the aggregate evaluation JSON. Must match the evaluation_id in the aggregate file.", "required": "yes" }, { "id": "eee_instance_level_eval:model_id", "source": "eee_instance_level_eval", "section": "model_id", "field": "", "schemaPath": "model_id", "fullPath": "eee_instance_level_eval.model_id", "type": "string", "description": "Identifier of the model in HuggingFace format (e.g. meta-llama/Llama-3.2-1B-Instruct)", "required": "yes" }, { "id": "eee_instance_level_eval:evaluation_name", "source": "eee_instance_level_eval", "section": "evaluation_name", "field": "", "schemaPath": "evaluation_name", "fullPath": "eee_instance_level_eval.evaluation_name", "type": "string", "description": "The specific eval name, ideally unique (e.g. GSM8K, mmlu_physics). Primarily for display/filtering when evaluation_result_id is unavailable. If an evaluation run reports multiple aggregate results/metrics for the same sample, emit multiple instance records (one per aggregate result) rather than trying to attach a single instance record to many aggregate results.", "required": "yes" }, { "id": "eee_instance_level_eval:evaluation_result_id", "source": "eee_instance_level_eval", "section": "evaluation_result_id", "field": "", "schemaPath": "evaluation_result_id", "fullPath": "eee_instance_level_eval.evaluation_result_id", "type": "string", "description": "Preferred deterministic foreign key to aggregate evaluation_results[].evaluation_result_id. This is intended to point to exactly one aggregate evaluation result; if a single underlying interaction/sample contributes to multiple aggregate results, emit multiple instance records with different evaluation_result_id values.", "required": "no" }, { "id": "eee_instance_level_eval:sample_id", "source": "eee_instance_level_eval", "section": "sample_id", "field": "", "schemaPath": "sample_id", "fullPath": "eee_instance_level_eval.sample_id", "type": "string", "description": "Question/sample identifier from the original dataset (e.g. gsm8k_0001)", "required": "yes" }, { "id": "eee_instance_level_eval:sample_hash", "source": "eee_instance_level_eval", "section": "sample_hash", "field": "", "schemaPath": "sample_hash", "fullPath": "eee_instance_level_eval.sample_hash", "type": "string | null", "description": "Hash of (input.raw + input.reference) to ensure comparison is between the same sample across models, in case sample_id is not consistent", "required": "no" }, { "id": "eee_instance_level_eval:interaction_type", "source": "eee_instance_level_eval", "section": "interaction_type", "field": "", "schemaPath": "interaction_type", "fullPath": "eee_instance_level_eval.interaction_type", "type": "enum: \"single_turn\" | \"multi_turn\" | \"agentic\"", "description": "Type of interaction: single_turn for simple Q&A, multi_turn for conversations, agentic for tool-using agents", "required": "yes" }, { "id": "eee_instance_level_eval:input", "source": "eee_instance_level_eval", "section": "input", "field": "", "schemaPath": "input", "fullPath": "eee_instance_level_eval.input", "type": "object", "description": "Input data for the evaluation sample", "required": "yes" }, { "id": "eee_instance_level_eval:input.raw", "source": "eee_instance_level_eval", "section": "input", "field": "raw", "schemaPath": "input.raw", "fullPath": "eee_instance_level_eval.input.raw", "type": "string", "description": "The raw input as defined in the eval", "required": "yes" }, { "id": "eee_instance_level_eval:input.formatted", "source": "eee_instance_level_eval", "section": "input", "field": "formatted", "schemaPath": "input.formatted", "fullPath": "eee_instance_level_eval.input.formatted", "type": "string | null", "description": "Includes chat template, CoT and all relevant modifications - basically the un-tokenized version of what the model sees", "required": "no" }, { "id": "eee_instance_level_eval:input.reference", "source": "eee_instance_level_eval", "section": "input", "field": "reference", "schemaPath": "input.reference", "fullPath": "eee_instance_level_eval.input.reference", "type": "array[string]", "description": "Ground truths or reference answers for comparison/scoring", "required": "yes" }, { "id": "eee_instance_level_eval:input.choices", "source": "eee_instance_level_eval", "section": "input", "field": "choices", "schemaPath": "input.choices", "fullPath": "eee_instance_level_eval.input.choices", "type": "array[string] | null", "description": "Optional list of choices for multiple-choice questions", "required": "no" }, { "id": "eee_instance_level_eval:output", "source": "eee_instance_level_eval", "section": "output", "field": "", "schemaPath": "output", "fullPath": "eee_instance_level_eval.output", "type": "object | null", "description": "Output data - only used for single_turn interactions, null for multi_turn/agentic", "required": "conditional" }, { "id": "eee_instance_level_eval:output.raw", "source": "eee_instance_level_eval", "section": "output", "field": "raw", "schemaPath": "output.raw", "fullPath": "eee_instance_level_eval.output.raw", "type": "array[string]", "description": "Complete model responses", "required": "yes" }, { "id": "eee_instance_level_eval:output.reasoning_trace", "source": "eee_instance_level_eval", "section": "output", "field": "reasoning_trace", "schemaPath": "output.reasoning_trace", "fullPath": "eee_instance_level_eval.output.reasoning_trace", "type": "array[string] | null", "description": "Reasoning traces of the model if applicable (e.g. chain-of-thought tokens)", "required": "no" }, { "id": "eee_instance_level_eval:messages", "source": "eee_instance_level_eval", "section": "messages", "field": "", "schemaPath": "messages", "fullPath": "eee_instance_level_eval.messages", "type": "array[object] | null", "description": "Full message transcript - used for multi_turn and agentic, null for single_turn. Contains all system, user, assistant, and tool messages in order.", "required": "conditional" }, { "id": "eee_instance_level_eval:messages.turn_idx", "source": "eee_instance_level_eval", "section": "messages", "field": "turn_idx", "schemaPath": "messages.turn_idx", "fullPath": "eee_instance_level_eval.messages.turn_idx", "type": "integer", "description": "Index starting from 0 indicating the position in the conversation", "required": "yes" }, { "id": "eee_instance_level_eval:messages.role", "source": "eee_instance_level_eval", "section": "messages", "field": "role", "schemaPath": "messages.role", "fullPath": "eee_instance_level_eval.messages.role", "type": "string", "description": "Role of the speaker (e.g. user, assistant, system, tool)", "required": "yes" }, { "id": "eee_instance_level_eval:messages.content", "source": "eee_instance_level_eval", "section": "messages", "field": "content", "schemaPath": "messages.content", "fullPath": "eee_instance_level_eval.messages.content", "type": "string | null", "description": "The actual raw text for that particular turn (can be null if empty)", "required": "no" }, { "id": "eee_instance_level_eval:messages.reasoning_trace", "source": "eee_instance_level_eval", "section": "messages", "field": "reasoning_trace", "schemaPath": "messages.reasoning_trace", "fullPath": "eee_instance_level_eval.messages.reasoning_trace", "type": "string | null", "description": "Reasoning trace for that particular turn if applicable", "required": "no" }, { "id": "eee_instance_level_eval:messages.tool_calls", "source": "eee_instance_level_eval", "section": "messages", "field": "tool_calls", "schemaPath": "messages.tool_calls", "fullPath": "eee_instance_level_eval.messages.tool_calls", "type": "array[object] | null", "description": "List of tool invocations for this turn, if applicable", "required": "no" }, { "id": "eee_instance_level_eval:messages.tool_calls[].id", "source": "eee_instance_level_eval", "section": "messages", "field": "tool_calls[].id", "schemaPath": "messages.tool_calls[].id", "fullPath": "eee_instance_level_eval.messages.tool_calls[].id", "type": "string", "description": "Unique identifier for the tool call", "required": "yes" }, { "id": "eee_instance_level_eval:messages.tool_calls[].name", "source": "eee_instance_level_eval", "section": "messages", "field": "tool_calls[].name", "schemaPath": "messages.tool_calls[].name", "fullPath": "eee_instance_level_eval.messages.tool_calls[].name", "type": "string", "description": "Name of tool/function", "required": "yes" }, { "id": "eee_instance_level_eval:messages.tool_calls[].arguments", "source": "eee_instance_level_eval", "section": "messages", "field": "tool_calls[].arguments", "schemaPath": "messages.tool_calls[].arguments", "fullPath": "eee_instance_level_eval.messages.tool_calls[].arguments", "type": "object | null", "description": "Arguments used to call the tool (all values must be strings)", "required": "no" }, { "id": "eee_instance_level_eval:messages.tool_call_id", "source": "eee_instance_level_eval", "section": "messages", "field": "tool_call_id", "schemaPath": "messages.tool_call_id", "fullPath": "eee_instance_level_eval.messages.tool_call_id", "type": "array[string] | null", "description": "Reference to the tool call ID(s) this message has the content payload for.", "required": "no" }, { "id": "eee_instance_level_eval:answer_attribution", "source": "eee_instance_level_eval", "section": "answer_attribution", "field": "", "schemaPath": "answer_attribution", "fullPath": "eee_instance_level_eval.answer_attribution", "type": "array[object]", "description": "Information about how the answer was extracted from the model output", "required": "yes" }, { "id": "eee_instance_level_eval:answer_attribution.turn_idx", "source": "eee_instance_level_eval", "section": "answer_attribution", "field": "turn_idx", "schemaPath": "answer_attribution.turn_idx", "fullPath": "eee_instance_level_eval.answer_attribution.turn_idx", "type": "integer", "description": "Turn index in messages. 0 for single_turn", "required": "yes" }, { "id": "eee_instance_level_eval:answer_attribution.source", "source": "eee_instance_level_eval", "section": "answer_attribution", "field": "source", "schemaPath": "answer_attribution.source", "fullPath": "eee_instance_level_eval.answer_attribution.source", "type": "string", "description": "Source of the extracted value (e.g. 'output.raw' or 'messages[turn_idx].content')", "required": "yes" }, { "id": "eee_instance_level_eval:answer_attribution.extracted_value", "source": "eee_instance_level_eval", "section": "answer_attribution", "field": "extracted_value", "schemaPath": "answer_attribution.extracted_value", "fullPath": "eee_instance_level_eval.answer_attribution.extracted_value", "type": "string", "description": "Value that was extracted", "required": "yes" }, { "id": "eee_instance_level_eval:answer_attribution.extraction_method", "source": "eee_instance_level_eval", "section": "answer_attribution", "field": "extraction_method", "schemaPath": "answer_attribution.extraction_method", "fullPath": "eee_instance_level_eval.answer_attribution.extraction_method", "type": "string", "description": "Method used to extract the value (e.g. regex, exact_match, llm_judge, custom)", "required": "yes" }, { "id": "eee_instance_level_eval:answer_attribution.is_terminal", "source": "eee_instance_level_eval", "section": "answer_attribution", "field": "is_terminal", "schemaPath": "answer_attribution.is_terminal", "fullPath": "eee_instance_level_eval.answer_attribution.is_terminal", "type": "boolean", "description": "Whether this is the final answer (false if intermediate outputs are used to build up to a final answer)", "required": "yes" }, { "id": "eee_instance_level_eval:evaluation", "source": "eee_instance_level_eval", "section": "evaluation", "field": "", "schemaPath": "evaluation", "fullPath": "eee_instance_level_eval.evaluation", "type": "object", "description": "Evaluation results and scoring data", "required": "yes" }, { "id": "eee_instance_level_eval:evaluation.score", "source": "eee_instance_level_eval", "section": "evaluation", "field": "score", "schemaPath": "evaluation.score", "fullPath": "eee_instance_level_eval.evaluation.score", "type": "number", "description": "Instance-level score", "required": "yes" }, { "id": "eee_instance_level_eval:evaluation.is_correct", "source": "eee_instance_level_eval", "section": "evaluation", "field": "is_correct", "schemaPath": "evaluation.is_correct", "fullPath": "eee_instance_level_eval.evaluation.is_correct", "type": "boolean", "description": "Whether the final answer is correct", "required": "yes" }, { "id": "eee_instance_level_eval:evaluation.num_turns", "source": "eee_instance_level_eval", "section": "evaluation", "field": "num_turns", "schemaPath": "evaluation.num_turns", "fullPath": "eee_instance_level_eval.evaluation.num_turns", "type": "integer | null", "description": "Number of turns in the interaction", "required": "no" }, { "id": "eee_instance_level_eval:evaluation.tool_calls_count", "source": "eee_instance_level_eval", "section": "evaluation", "field": "tool_calls_count", "schemaPath": "evaluation.tool_calls_count", "fullPath": "eee_instance_level_eval.evaluation.tool_calls_count", "type": "integer | null", "description": "Count of tool calls across all turns in messages", "required": "no" }, { "id": "eee_instance_level_eval:token_usage", "source": "eee_instance_level_eval", "section": "token_usage", "field": "", "schemaPath": "token_usage", "fullPath": "eee_instance_level_eval.token_usage", "type": "object | null", "description": "Token usage for the model completion", "required": "no" }, { "id": "eee_instance_level_eval:token_usage.input_tokens", "source": "eee_instance_level_eval", "section": "token_usage", "field": "input_tokens", "schemaPath": "token_usage.input_tokens", "fullPath": "eee_instance_level_eval.token_usage.input_tokens", "type": "integer", "description": "Total input tokens used", "required": "yes" }, { "id": "eee_instance_level_eval:token_usage.output_tokens", "source": "eee_instance_level_eval", "section": "token_usage", "field": "output_tokens", "schemaPath": "token_usage.output_tokens", "fullPath": "eee_instance_level_eval.token_usage.output_tokens", "type": "integer", "description": "Total output tokens used", "required": "yes" }, { "id": "eee_instance_level_eval:token_usage.total_tokens", "source": "eee_instance_level_eval", "section": "token_usage", "field": "total_tokens", "schemaPath": "token_usage.total_tokens", "fullPath": "eee_instance_level_eval.token_usage.total_tokens", "type": "integer", "description": "Total tokens used", "required": "yes" }, { "id": "eee_instance_level_eval:token_usage.input_tokens_cache_write", "source": "eee_instance_level_eval", "section": "token_usage", "field": "input_tokens_cache_write", "schemaPath": "token_usage.input_tokens_cache_write", "fullPath": "eee_instance_level_eval.token_usage.input_tokens_cache_write", "type": "integer | null", "description": "Number of tokens written to the cache", "required": "no" }, { "id": "eee_instance_level_eval:token_usage.input_tokens_cache_read", "source": "eee_instance_level_eval", "section": "token_usage", "field": "input_tokens_cache_read", "schemaPath": "token_usage.input_tokens_cache_read", "fullPath": "eee_instance_level_eval.token_usage.input_tokens_cache_read", "type": "integer | null", "description": "Number of tokens retrieved from the cache", "required": "no" }, { "id": "eee_instance_level_eval:token_usage.reasoning_tokens", "source": "eee_instance_level_eval", "section": "token_usage", "field": "reasoning_tokens", "schemaPath": "token_usage.reasoning_tokens", "fullPath": "eee_instance_level_eval.token_usage.reasoning_tokens", "type": "integer | null", "description": "Number of tokens used for reasoning", "required": "no" }, { "id": "eee_instance_level_eval:performance", "source": "eee_instance_level_eval", "section": "performance", "field": "", "schemaPath": "performance", "fullPath": "eee_instance_level_eval.performance", "type": "object | null", "description": "Performance and latency metrics", "required": "no" }, { "id": "eee_instance_level_eval:performance.latency_ms", "source": "eee_instance_level_eval", "section": "performance", "field": "latency_ms", "schemaPath": "performance.latency_ms", "fullPath": "eee_instance_level_eval.performance.latency_ms", "type": "number | null", "description": "Total latency in milliseconds", "required": "no" }, { "id": "eee_instance_level_eval:performance.time_to_first_token_ms", "source": "eee_instance_level_eval", "section": "performance", "field": "time_to_first_token_ms", "schemaPath": "performance.time_to_first_token_ms", "fullPath": "eee_instance_level_eval.performance.time_to_first_token_ms", "type": "number | null", "description": "Time to first token in milliseconds", "required": "no" }, { "id": "eee_instance_level_eval:performance.generation_time_ms", "source": "eee_instance_level_eval", "section": "performance", "field": "generation_time_ms", "schemaPath": "performance.generation_time_ms", "fullPath": "eee_instance_level_eval.performance.generation_time_ms", "type": "number | null", "description": "Time for generation in milliseconds", "required": "no" }, { "id": "eee_instance_level_eval:performance.additional_details", "source": "eee_instance_level_eval", "section": "performance", "field": "additional_details", "schemaPath": "performance.additional_details", "fullPath": "eee_instance_level_eval.performance.additional_details", "type": "object | null", "description": "Additional performance metrics (key-value pairs, all values must be strings)", "required": "no" }, { "id": "eee_instance_level_eval:error", "source": "eee_instance_level_eval", "section": "error", "field": "", "schemaPath": "error", "fullPath": "eee_instance_level_eval.error", "type": "string | null", "description": "Information about any error that occurred (e.g. timeout, refusal, API error)", "required": "no" }, { "id": "eee_instance_level_eval:metadata", "source": "eee_instance_level_eval", "section": "metadata", "field": "", "schemaPath": "metadata", "fullPath": "eee_instance_level_eval.metadata", "type": "object | null", "description": "Optional metadata about the sample (e.g. subject, difficulty, tags)", "required": "no" } ]