Spaces:
Running
Running
| { | |
| "eval_summary_id": "apex_v1", | |
| "benchmark": "APEX v1", | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "benchmark_component_key": "medicine_md", | |
| "benchmark_component_name": "Medicine (MD)", | |
| "evaluation_name": "APEX v1", | |
| "display_name": "APEX v1", | |
| "canonical_display_name": "APEX v1", | |
| "is_summary_score": false, | |
| "category": "knowledge", | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "benchmark_card": { | |
| "benchmark_details": { | |
| "name": "APEX-v1", | |
| "overview": "APEX-Agents (AI Productivity Index for Agents) measures the ability of AI agents to execute long-horizon, cross-application tasks created by investment banking analysts, management consultants, and corporate lawyers. The benchmark contains 480 tasks and requires agents to navigate realistic work environments with files and tools.", | |
| "data_type": "text", | |
| "domains": [ | |
| "investment banking", | |
| "management consulting", | |
| "corporate law", | |
| "finance", | |
| "legal", | |
| "consulting" | |
| ], | |
| "languages": [ | |
| "English" | |
| ], | |
| "similar_benchmarks": [ | |
| "Not specified" | |
| ], | |
| "resources": [ | |
| "https://arxiv.org/abs/2601.14242", | |
| "https://huggingface.co/datasets/Mercor/APEX-v1" | |
| ], | |
| "benchmark_type": "single" | |
| }, | |
| "purpose_and_intended_users": { | |
| "goal": "To assess whether AI agents can reliably execute highly complex professional services work, bridging the gap between existing agentic evaluations and real-world professional workflows.", | |
| "audience": [ | |
| "AI researchers", | |
| "Developers working on agentic systems" | |
| ], | |
| "tasks": [ | |
| "Text generation", | |
| "Question answering", | |
| "Reasoning", | |
| "Demonstrating advanced knowledge", | |
| "Using multiple applications", | |
| "Planning over long horizons within realistic project scenarios" | |
| ], | |
| "limitations": "Differences in benchmark scores below 1 percentage point should be interpreted cautiously due to a small error rate in the automated grading system (1.9% false negative rate and 1.3% false positive rate).", | |
| "out_of_scope_uses": [ | |
| "Not specified" | |
| ] | |
| }, | |
| "data": { | |
| "source": "The benchmark data was created by industry professionals including investment banking analysts, management consultants, and corporate lawyers. These professionals were organized into teams, assigned specific roles, and tasked with delivering complete projects over 5-10 day periods, producing high-quality customer-ready deliverables from scratch.", | |
| "size": "480 tasks", | |
| "format": "The specific structure of individual data instances is not described", | |
| "annotation": "Tasks were created by professionals using files from within each project environment. A baselining study was conducted where independent experts executed 20% of tasks (96 tasks) to verify task feasibility, rubric fairness, and time estimates." | |
| }, | |
| "methodology": { | |
| "methods": [ | |
| "Models are evaluated using agent execution in realistic environments", | |
| "Eight trajectories are collected for each agent-task pair, with each trajectory scored as pass or fail" | |
| ], | |
| "metrics": [ | |
| "Pass@1 (task-uniform mean of per-task pass rates)", | |
| "Pass@8 (passing at least once in eight attempts)", | |
| "Pass^8 (passing consistently on all eight attempts)" | |
| ], | |
| "calculation": "The overall Pass@1 score is computed as the task-uniform mean of per-task pass rates across all 480 tasks. Confidence intervals are calculated using task-level bootstrapping with 10,000 resamples", | |
| "interpretation": "Higher Pass@1 scores indicate better performance", | |
| "baseline_results": "Gemini 3 Flash (Thinking=High): 24.0%, GPT-5.2 (Thinking=High): 23.0%, Claude Opus 4.5 (Thinking=High): [score not specified], Gemini 3 Pro (Thinking=High): [score not specified], GPT-OSS-120B (High): 15.2%, Grok 4: 0%", | |
| "validation": "Automated evaluation used a judge model with 98.5% accuracy against human-labeled ground truth. A baselining study with experts validated task feasibility and rubric fairness" | |
| }, | |
| "ethical_and_legal_considerations": { | |
| "privacy_and_anonymity": "Not specified", | |
| "data_licensing": "Creative Commons Attribution 4.0", | |
| "consent_procedures": "Not specified", | |
| "compliance_with_regulations": "Not specified" | |
| }, | |
| "possible_risks": [ | |
| { | |
| "category": "Over- or under-reliance", | |
| "description": [ | |
| "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should." | |
| ], | |
| "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html" | |
| }, | |
| { | |
| "category": "Unrepresentative data", | |
| "description": [ | |
| "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios." | |
| ], | |
| "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html" | |
| }, | |
| { | |
| "category": "Incomplete AI agent evaluation", | |
| "description": [ | |
| "Evaluating the performance or accuracy or an agent is difficult because of system complexity and open-endedness." | |
| ], | |
| "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/incomplete-ai-agent-evaluation-agentic.html" | |
| }, | |
| { | |
| "category": "Reproducibility", | |
| "description": [ | |
| "Replicating agent behavior or output can be impacted by changes or updates made to external services and tools. This impact is increased if the agent is built with generative AI." | |
| ], | |
| "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/reproducibility-agentic.html" | |
| }, | |
| { | |
| "category": "Improper usage", | |
| "description": [ | |
| "Improper usage occurs when a model is used for a purpose that it was not originally designed for." | |
| ], | |
| "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html" | |
| } | |
| ], | |
| "flagged_fields": {}, | |
| "missing_fields": [ | |
| "benchmark_details.similar_benchmarks", | |
| "purpose_and_intended_users.out_of_scope_uses", | |
| "ethical_and_legal_considerations.privacy_and_anonymity", | |
| "ethical_and_legal_considerations.consent_procedures", | |
| "ethical_and_legal_considerations.compliance_with_regulations" | |
| ], | |
| "card_info": { | |
| "created_at": "2026-04-14T14:28:12.501639", | |
| "llm": "deepseek-ai/DeepSeek-V3.1" | |
| } | |
| }, | |
| "tags": { | |
| "domains": [ | |
| "investment banking", | |
| "management consulting", | |
| "corporate law", | |
| "finance", | |
| "legal", | |
| "consulting" | |
| ], | |
| "languages": [ | |
| "English" | |
| ], | |
| "tasks": [ | |
| "Text generation", | |
| "Question answering", | |
| "Reasoning", | |
| "Demonstrating advanced knowledge", | |
| "Using multiple applications", | |
| "Planning over long horizons within realistic project scenarios" | |
| ] | |
| }, | |
| "subtasks": [ | |
| { | |
| "subtask_key": "big_law", | |
| "subtask_name": "Big Law", | |
| "display_name": "Big Law", | |
| "metrics": [ | |
| { | |
| "metric_summary_id": "apex_v1_big_law_score", | |
| "legacy_eval_summary_id": "apex_v1_big_law", | |
| "evaluation_name": "Big Law", | |
| "display_name": "APEX v1 / Big Law / Score", | |
| "canonical_display_name": "APEX v1 / Big Law / Score", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "big_law", | |
| "slice_name": "Big Law", | |
| "lower_is_better": false, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "metric_config": { | |
| "evaluation_description": "Big law associate score.", | |
| "lower_is_better": false, | |
| "score_type": "continuous", | |
| "min_score": 0, | |
| "max_score": 1, | |
| "additional_details": { | |
| "raw_evaluation_name": "Big Law Score" | |
| }, | |
| "metric_id": "apex_v1.score", | |
| "metric_name": "Score", | |
| "metric_kind": "score", | |
| "metric_unit": "proportion" | |
| }, | |
| "model_results": [ | |
| { | |
| "model_id": "openai/gpt-5", | |
| "model_route_id": "openai__gpt-5", | |
| "model_name": "GPT 5", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 5", | |
| "score": 0.78, | |
| "evaluation_id": "apex-v1/openai_gpt-5/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5/apex_v1_openai_gpt_5_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "big_law", | |
| "benchmark_component_name": "Big Law", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "big_law", | |
| "slice_name": "Big Law", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Big Law / Score", | |
| "canonical_display_name": "APEX v1 / Big Law / Score", | |
| "raw_evaluation_name": "Big Law", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "openai/gpt-5-1", | |
| "model_route_id": "openai__gpt-5-1", | |
| "model_name": "GPT 5.1", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 5.1", | |
| "score": 0.77, | |
| "evaluation_id": "apex-v1/openai_gpt-5.1/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-1/apex_v1_openai_gpt_5_1_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "big_law", | |
| "benchmark_component_name": "Big Law", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "big_law", | |
| "slice_name": "Big Law", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Big Law / Score", | |
| "canonical_display_name": "APEX v1 / Big Law / Score", | |
| "raw_evaluation_name": "Big Law", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "openai/o3", | |
| "model_route_id": "openai__o3", | |
| "model_name": "o3", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/o3", | |
| "score": 0.76, | |
| "evaluation_id": "apex-v1/openai_o3/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__o3/apex_v1_openai_o3_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "big_law", | |
| "benchmark_component_name": "Big Law", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "big_law", | |
| "slice_name": "Big Law", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Big Law / Score", | |
| "canonical_display_name": "APEX v1 / Big Law / Score", | |
| "raw_evaluation_name": "Big Law", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| } | |
| ], | |
| "models_count": 3, | |
| "top_score": 0.78 | |
| } | |
| ], | |
| "metrics_count": 1, | |
| "metric_names": [ | |
| "Score" | |
| ] | |
| }, | |
| { | |
| "subtask_key": "consulting", | |
| "subtask_name": "Consulting", | |
| "display_name": "Consulting", | |
| "metrics": [ | |
| { | |
| "metric_summary_id": "apex_v1_consulting_score", | |
| "legacy_eval_summary_id": "apex_v1_consulting", | |
| "evaluation_name": "Consulting", | |
| "display_name": "APEX v1 / Consulting / Score", | |
| "canonical_display_name": "APEX v1 / Consulting / Score", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "consulting", | |
| "slice_name": "Consulting", | |
| "lower_is_better": false, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "metric_config": { | |
| "evaluation_description": "Management consulting score.", | |
| "lower_is_better": false, | |
| "score_type": "continuous", | |
| "min_score": 0, | |
| "max_score": 1, | |
| "additional_details": { | |
| "raw_evaluation_name": "Consulting Score" | |
| }, | |
| "metric_id": "apex_v1.score", | |
| "metric_name": "Score", | |
| "metric_kind": "score", | |
| "metric_unit": "proportion" | |
| }, | |
| "model_results": [ | |
| { | |
| "model_id": "openai/gpt-5-2-pro", | |
| "model_route_id": "openai__gpt-5-2-pro", | |
| "model_name": "GPT 5.2 Pro", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 5.2 Pro", | |
| "score": 0.64, | |
| "evaluation_id": "apex-v1/openai_gpt-5.2-pro/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2-pro/apex_v1_openai_gpt_5_2_pro_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "consulting", | |
| "benchmark_component_name": "Consulting", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "consulting", | |
| "slice_name": "Consulting", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Consulting / Score", | |
| "canonical_display_name": "APEX v1 / Consulting / Score", | |
| "raw_evaluation_name": "Consulting", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "google/gemini-3-pro", | |
| "model_route_id": "google__gemini-3-pro", | |
| "model_name": "Gemini 3 Pro", | |
| "developer": "google", | |
| "variant_key": "default", | |
| "raw_model_id": "google/Gemini 3 Pro", | |
| "score": 0.64, | |
| "evaluation_id": "apex-v1/google_gemini-3-pro/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro/apex_v1_google_gemini_3_pro_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "consulting", | |
| "benchmark_component_name": "Consulting", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "consulting", | |
| "slice_name": "Consulting", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Consulting / Score", | |
| "canonical_display_name": "APEX v1 / Consulting / Score", | |
| "raw_evaluation_name": "Consulting", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "google/gemini-3-flash", | |
| "model_route_id": "google__gemini-3-flash", | |
| "model_name": "Gemini 3 Flash", | |
| "developer": "google", | |
| "variant_key": "default", | |
| "raw_model_id": "google/Gemini 3 Flash", | |
| "score": 0.64, | |
| "evaluation_id": "apex-v1/google_gemini-3-flash/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-flash/apex_v1_google_gemini_3_flash_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "consulting", | |
| "benchmark_component_name": "Consulting", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "consulting", | |
| "slice_name": "Consulting", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Consulting / Score", | |
| "canonical_display_name": "APEX v1 / Consulting / Score", | |
| "raw_evaluation_name": "Consulting", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| } | |
| ], | |
| "models_count": 3, | |
| "top_score": 0.64 | |
| } | |
| ], | |
| "metrics_count": 1, | |
| "metric_names": [ | |
| "Score" | |
| ] | |
| }, | |
| { | |
| "subtask_key": "investment_banking", | |
| "subtask_name": "Investment Banking", | |
| "display_name": "Investment Banking", | |
| "metrics": [ | |
| { | |
| "metric_summary_id": "apex_v1_investment_banking_score", | |
| "legacy_eval_summary_id": "apex_v1_investment_banking", | |
| "evaluation_name": "Investment Banking", | |
| "display_name": "APEX v1 / Investment Banking / Score", | |
| "canonical_display_name": "APEX v1 / Investment Banking / Score", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "investment_banking", | |
| "slice_name": "Investment Banking", | |
| "lower_is_better": false, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "metric_config": { | |
| "evaluation_description": "Investment banking associate score.", | |
| "lower_is_better": false, | |
| "score_type": "continuous", | |
| "min_score": 0, | |
| "max_score": 1, | |
| "additional_details": { | |
| "raw_evaluation_name": "Investment Banking Score" | |
| }, | |
| "metric_id": "apex_v1.score", | |
| "metric_name": "Score", | |
| "metric_kind": "score", | |
| "metric_unit": "proportion" | |
| }, | |
| "model_results": [ | |
| { | |
| "model_id": "openai/gpt-5-2-pro", | |
| "model_route_id": "openai__gpt-5-2-pro", | |
| "model_name": "GPT 5.2 Pro", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 5.2 Pro", | |
| "score": 0.64, | |
| "evaluation_id": "apex-v1/openai_gpt-5.2-pro/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2-pro/apex_v1_openai_gpt_5_2_pro_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "investment_banking", | |
| "benchmark_component_name": "Investment Banking", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "investment_banking", | |
| "slice_name": "Investment Banking", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Investment Banking / Score", | |
| "canonical_display_name": "APEX v1 / Investment Banking / Score", | |
| "raw_evaluation_name": "Investment Banking", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "google/gemini-3-pro", | |
| "model_route_id": "google__gemini-3-pro", | |
| "model_name": "Gemini 3 Pro", | |
| "developer": "google", | |
| "variant_key": "default", | |
| "raw_model_id": "google/Gemini 3 Pro", | |
| "score": 0.63, | |
| "evaluation_id": "apex-v1/google_gemini-3-pro/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro/apex_v1_google_gemini_3_pro_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "investment_banking", | |
| "benchmark_component_name": "Investment Banking", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "investment_banking", | |
| "slice_name": "Investment Banking", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Investment Banking / Score", | |
| "canonical_display_name": "APEX v1 / Investment Banking / Score", | |
| "raw_evaluation_name": "Investment Banking", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "openai/gpt-5", | |
| "model_route_id": "openai__gpt-5", | |
| "model_name": "GPT 5", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 5", | |
| "score": 0.61, | |
| "evaluation_id": "apex-v1/openai_gpt-5/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5/apex_v1_openai_gpt_5_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "investment_banking", | |
| "benchmark_component_name": "Investment Banking", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "investment_banking", | |
| "slice_name": "Investment Banking", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Investment Banking / Score", | |
| "canonical_display_name": "APEX v1 / Investment Banking / Score", | |
| "raw_evaluation_name": "Investment Banking", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| } | |
| ], | |
| "models_count": 3, | |
| "top_score": 0.64 | |
| } | |
| ], | |
| "metrics_count": 1, | |
| "metric_names": [ | |
| "Score" | |
| ] | |
| }, | |
| { | |
| "subtask_key": "medicine_md", | |
| "subtask_name": "Medicine (MD)", | |
| "display_name": "Medicine (MD)", | |
| "metrics": [ | |
| { | |
| "metric_summary_id": "apex_v1_medicine_md_score", | |
| "legacy_eval_summary_id": "apex_v1_medicine_md", | |
| "evaluation_name": "Medicine (MD)", | |
| "display_name": "APEX v1 / Medicine (MD) / Score", | |
| "canonical_display_name": "APEX v1 / Medicine (MD) / Score", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "medicine_md", | |
| "slice_name": "Medicine (MD)", | |
| "lower_is_better": false, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "metric_config": { | |
| "evaluation_description": "Primary care physician (MD) score.", | |
| "lower_is_better": false, | |
| "score_type": "continuous", | |
| "min_score": 0, | |
| "max_score": 1, | |
| "additional_details": { | |
| "raw_evaluation_name": "Medicine (MD) Score" | |
| }, | |
| "metric_id": "apex_v1.score", | |
| "metric_name": "Score", | |
| "metric_kind": "score", | |
| "metric_unit": "proportion" | |
| }, | |
| "model_results": [ | |
| { | |
| "model_id": "openai/gpt-5", | |
| "model_route_id": "openai__gpt-5", | |
| "model_name": "GPT 5", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 5", | |
| "score": 0.66, | |
| "evaluation_id": "apex-v1/openai_gpt-5/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5/apex_v1_openai_gpt_5_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "medicine_md", | |
| "benchmark_component_name": "Medicine (MD)", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "medicine_md", | |
| "slice_name": "Medicine (MD)", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Medicine (MD) / Score", | |
| "canonical_display_name": "APEX v1 / Medicine (MD) / Score", | |
| "raw_evaluation_name": "Medicine (MD)", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "openai/gpt-5-2-pro", | |
| "model_route_id": "openai__gpt-5-2-pro", | |
| "model_name": "GPT 5.2 Pro", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 5.2 Pro", | |
| "score": 0.65, | |
| "evaluation_id": "apex-v1/openai_gpt-5.2-pro/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2-pro/apex_v1_openai_gpt_5_2_pro_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "medicine_md", | |
| "benchmark_component_name": "Medicine (MD)", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "medicine_md", | |
| "slice_name": "Medicine (MD)", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Medicine (MD) / Score", | |
| "canonical_display_name": "APEX v1 / Medicine (MD) / Score", | |
| "raw_evaluation_name": "Medicine (MD)", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "anthropic/opus-4-5", | |
| "model_route_id": "anthropic__opus-4-5", | |
| "model_name": "Opus 4.5", | |
| "developer": "anthropic", | |
| "variant_key": "default", | |
| "raw_model_id": "anthropic/Opus 4.5", | |
| "score": 0.65, | |
| "evaluation_id": "apex-v1/anthropic_opus-4.5/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/anthropic__opus-4-5/apex_v1_anthropic_opus_4_5_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": "medicine_md", | |
| "benchmark_component_name": "Medicine (MD)", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": "medicine_md", | |
| "slice_name": "Medicine (MD)", | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Medicine (MD) / Score", | |
| "canonical_display_name": "APEX v1 / Medicine (MD) / Score", | |
| "raw_evaluation_name": "Medicine (MD)", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| } | |
| ], | |
| "models_count": 3, | |
| "top_score": 0.66 | |
| } | |
| ], | |
| "metrics_count": 1, | |
| "metric_names": [ | |
| "Score" | |
| ] | |
| } | |
| ], | |
| "metrics": [ | |
| { | |
| "metric_summary_id": "apex_v1_score", | |
| "legacy_eval_summary_id": "apex_v1_apex_v1", | |
| "evaluation_name": "apex-v1", | |
| "display_name": "APEX v1 / Score", | |
| "canonical_display_name": "APEX v1 / Score", | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": null, | |
| "slice_name": null, | |
| "lower_is_better": false, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "metric_config": { | |
| "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", | |
| "lower_is_better": false, | |
| "score_type": "continuous", | |
| "min_score": 0, | |
| "max_score": 1, | |
| "additional_details": { | |
| "raw_evaluation_name": "Overall Score" | |
| }, | |
| "metric_id": "apex_v1.score", | |
| "metric_name": "Score", | |
| "metric_kind": "score", | |
| "metric_unit": "proportion" | |
| }, | |
| "model_results": [ | |
| { | |
| "model_id": "openai/gpt-5", | |
| "model_route_id": "openai__gpt-5", | |
| "model_name": "GPT 5", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 5", | |
| "score": 0.67, | |
| "evaluation_id": "apex-v1/openai_gpt-5/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5/apex_v1_openai_gpt_5_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": null, | |
| "benchmark_component_name": null, | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": null, | |
| "slice_name": null, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Score", | |
| "canonical_display_name": "APEX v1 / Score", | |
| "raw_evaluation_name": "apex-v1", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "openai/gpt-5-2-pro", | |
| "model_route_id": "openai__gpt-5-2-pro", | |
| "model_name": "GPT 5.2 Pro", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 5.2 Pro", | |
| "score": 0.668, | |
| "evaluation_id": "apex-v1/openai_gpt-5.2-pro/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-5-2-pro/apex_v1_openai_gpt_5_2_pro_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": null, | |
| "benchmark_component_name": null, | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": null, | |
| "slice_name": null, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Score", | |
| "canonical_display_name": "APEX v1 / Score", | |
| "raw_evaluation_name": "apex-v1", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "google/gemini-3-pro", | |
| "model_route_id": "google__gemini-3-pro", | |
| "model_name": "Gemini 3 Pro", | |
| "developer": "google", | |
| "variant_key": "default", | |
| "raw_model_id": "google/Gemini 3 Pro", | |
| "score": 0.643, | |
| "evaluation_id": "apex-v1/google_gemini-3-pro/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-pro/apex_v1_google_gemini_3_pro_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": null, | |
| "benchmark_component_name": null, | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": null, | |
| "slice_name": null, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Score", | |
| "canonical_display_name": "APEX v1 / Score", | |
| "raw_evaluation_name": "apex-v1", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "google/gemini-3-flash", | |
| "model_route_id": "google__gemini-3-flash", | |
| "model_name": "Gemini 3 Flash", | |
| "developer": "google", | |
| "variant_key": "default", | |
| "raw_model_id": "google/Gemini 3 Flash", | |
| "score": 0.64, | |
| "evaluation_id": "apex-v1/google_gemini-3-flash/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-3-flash/apex_v1_google_gemini_3_flash_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": null, | |
| "benchmark_component_name": null, | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": null, | |
| "slice_name": null, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Score", | |
| "canonical_display_name": "APEX v1 / Score", | |
| "raw_evaluation_name": "apex-v1", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "xai/grok-4", | |
| "model_route_id": "xai__grok-4", | |
| "model_name": "Grok 4", | |
| "developer": "xai", | |
| "variant_key": "default", | |
| "raw_model_id": "xai/Grok 4", | |
| "score": 0.635, | |
| "evaluation_id": "apex-v1/xai_grok-4/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/xai__grok-4/apex_v1_xai_grok_4_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": null, | |
| "benchmark_component_name": null, | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": null, | |
| "slice_name": null, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Score", | |
| "canonical_display_name": "APEX v1 / Score", | |
| "raw_evaluation_name": "apex-v1", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "google/gemini-2-5-flash", | |
| "model_route_id": "google__gemini-2-5-flash", | |
| "model_name": "Gemini 2.5 Flash", | |
| "developer": "google", | |
| "variant_key": "default", | |
| "raw_model_id": "google/Gemini 2.5 Flash", | |
| "score": 0.604, | |
| "evaluation_id": "apex-v1/google_gemini-2.5-flash/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/google__gemini-2-5-flash/apex_v1_google_gemini_2_5_flash_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": null, | |
| "benchmark_component_name": null, | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": null, | |
| "slice_name": null, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Score", | |
| "canonical_display_name": "APEX v1 / Score", | |
| "raw_evaluation_name": "apex-v1", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| }, | |
| { | |
| "model_id": "openai/gpt-4o", | |
| "model_route_id": "openai__gpt-4o", | |
| "model_name": "GPT 4o", | |
| "developer": "openai", | |
| "variant_key": "default", | |
| "raw_model_id": "openai/GPT 4o", | |
| "score": 0.359, | |
| "evaluation_id": "apex-v1/openai_gpt-4o/1773260200", | |
| "retrieved_timestamp": "1773260200", | |
| "source_metadata": { | |
| "source_name": "Mercor APEX-v1 Leaderboard", | |
| "source_type": "evaluation_run", | |
| "source_organization_name": "Mercor", | |
| "source_organization_url": "https://www.mercor.com", | |
| "evaluator_relationship": "first_party" | |
| }, | |
| "source_data": { | |
| "dataset_name": "apex-v1", | |
| "source_type": "hf_dataset", | |
| "hf_repo": "Mercor/APEX-v1" | |
| }, | |
| "source_record_url": "https://huggingface.co/datasets/evaleval/card_backend/resolve/main/records/openai__gpt-4o/apex_v1_openai_gpt_4o_1773260200.json", | |
| "detailed_evaluation_results": null, | |
| "detailed_evaluation_results_meta": null, | |
| "passthrough_top_level_fields": null, | |
| "instance_level_data": null, | |
| "normalized_result": { | |
| "benchmark_family_key": "apex_v1", | |
| "benchmark_family_name": "APEX v1", | |
| "benchmark_parent_key": "apex_v1", | |
| "benchmark_parent_name": "APEX v1", | |
| "benchmark_component_key": null, | |
| "benchmark_component_name": null, | |
| "benchmark_leaf_key": "apex_v1", | |
| "benchmark_leaf_name": "APEX v1", | |
| "slice_key": null, | |
| "slice_name": null, | |
| "metric_name": "Score", | |
| "metric_id": "apex_v1.score", | |
| "metric_key": "score", | |
| "metric_source": "metric_config", | |
| "display_name": "Score", | |
| "canonical_display_name": "APEX v1 / Score", | |
| "raw_evaluation_name": "apex-v1", | |
| "is_summary_score": false | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reproducibility_gap": { | |
| "has_reproducibility_gap": true, | |
| "missing_fields": [ | |
| "temperature", | |
| "max_tokens" | |
| ], | |
| "required_field_count": 2, | |
| "populated_field_count": 0, | |
| "signal_version": "1.0" | |
| }, | |
| "provenance": { | |
| "source_type": "first_party", | |
| "is_multi_source": false, | |
| "first_party_only": true, | |
| "distinct_reporting_organizations": 1, | |
| "signal_version": "1.0" | |
| }, | |
| "variant_divergence": null, | |
| "cross_party_divergence": null | |
| } | |
| } | |
| } | |
| ], | |
| "models_count": 7, | |
| "top_score": 0.67 | |
| } | |
| ], | |
| "subtasks_count": 4, | |
| "metrics_count": 5, | |
| "models_count": 10, | |
| "metric_names": [ | |
| "Score" | |
| ], | |
| "primary_metric_name": "Score", | |
| "top_score": null, | |
| "instance_data": { | |
| "available": false, | |
| "url_count": 0, | |
| "sample_urls": [], | |
| "models_with_loaded_instances": 0 | |
| }, | |
| "evalcards": { | |
| "annotations": { | |
| "reporting_completeness": { | |
| "completeness_score": 0.9285714285714286, | |
| "total_fields_evaluated": 28, | |
| "missing_required_fields": [ | |
| "evalcards.lifecycle_status", | |
| "evalcards.preregistration_url" | |
| ], | |
| "partial_fields": [], | |
| "field_scores": [ | |
| { | |
| "field_path": "autobenchmarkcard.benchmark_details.name", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.benchmark_details.overview", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.benchmark_details.data_type", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.benchmark_details.domains", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.benchmark_details.languages", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.benchmark_details.similar_benchmarks", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.benchmark_details.resources", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.purpose_and_intended_users.goal", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.purpose_and_intended_users.audience", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.purpose_and_intended_users.tasks", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.purpose_and_intended_users.limitations", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.purpose_and_intended_users.out_of_scope_uses", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.methodology.methods", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.methodology.metrics", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.methodology.calculation", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.methodology.interpretation", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.methodology.baseline_results", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.methodology.validation", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.ethical_and_legal_considerations.privacy_and_anonymity", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.ethical_and_legal_considerations.data_licensing", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.ethical_and_legal_considerations.consent_procedures", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.ethical_and_legal_considerations.compliance_with_regulations", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "autobenchmarkcard.data", | |
| "coverage_type": "partial", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "eee_eval.source_metadata.source_type", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "eee_eval.source_metadata.source_organization_name", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "eee_eval.source_metadata.evaluator_relationship", | |
| "coverage_type": "full", | |
| "score": 1.0 | |
| }, | |
| { | |
| "field_path": "evalcards.lifecycle_status", | |
| "coverage_type": "reserved", | |
| "score": 0.0 | |
| }, | |
| { | |
| "field_path": "evalcards.preregistration_url", | |
| "coverage_type": "reserved", | |
| "score": 0.0 | |
| } | |
| ], | |
| "signal_version": "1.0" | |
| }, | |
| "benchmark_comparability": { | |
| "variant_divergence_groups": [], | |
| "cross_party_divergence_groups": [] | |
| } | |
| } | |
| }, | |
| "reproducibility_summary": { | |
| "results_total": 19, | |
| "has_reproducibility_gap_count": 19, | |
| "populated_ratio_avg": 0.0 | |
| }, | |
| "provenance_summary": { | |
| "total_results": 19, | |
| "total_groups": 19, | |
| "multi_source_groups": 0, | |
| "first_party_only_groups": 19, | |
| "source_type_distribution": { | |
| "first_party": 19, | |
| "third_party": 0, | |
| "collaborative": 0, | |
| "unspecified": 0 | |
| } | |
| }, | |
| "comparability_summary": { | |
| "total_groups": 19, | |
| "groups_with_variant_check": 0, | |
| "groups_with_cross_party_check": 0, | |
| "variant_divergent_count": 0, | |
| "cross_party_divergent_count": 0 | |
| } | |
| } |