diff --git "a/public/benchmarks/meta-llama-3-70b.json" "b/public/benchmarks/meta-llama-3-70b.json" --- "a/public/benchmarks/meta-llama-3-70b.json" +++ "b/public/benchmarks/meta-llama-3-70b.json" @@ -1,7 +1,7 @@ { "schema_version": "0.1.0", "evaluation_id": "meta-llama-3-70b-1765899234257", - "retrieved_timestamp": "2025-12-16T15:33:54.257Z", + "retrieved_timestamp": "2024-08-10T16:46:19.575Z", "source_data": { "dataset_name": "Demo Benchmark Suite", "samples_number": 1000 @@ -35,9 +35,9 @@ }, "evaluation_results": [ { - "evaluation_name": "ADE20K", + "evaluation_name": "DigiData: Training and Evaluating General-Purpose Mobile Control Agents", "metric_config": { - "evaluation_description": "ADE20K Standard Accuracy", + "evaluation_description": "DigiData: Training and Evaluating General-Purpose Mobile Control Agents Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -45,38 +45,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8250870109200094, + "score": 0.6043264625935588, "details": { - "subtask_a": 0.8250870109200094, - "subtask_b": 0.8250870109200094 + "subtask_a": 0.6043264625935588, + "subtask_b": 0.6043264625935588 } }, "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Scene parsing, Semantic segmentation, Multi-scale recognition, Part segmentation", + "purpose": "Development; Research", + "principles_tested": "mobile control agents", "functional_props": "Core Performance", - "input_modality": "Vision (Image)", - "output_modality": "Structured Data", + "input_modality": "Text + Vision", + "output_modality": "Actions", "input_source": "New dataset (released with eval)", "output_source": "Human annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (20K), Val (2K), Test (3K)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives diverse scene image 2. Model predicts per-pixel semantic labels (150 classes) 3. Mean IoU and pixel accuracy computed", + "size": "Small (< 1K samples)", + "splits": "", + "design": "Dynamic data-driven (adaptive/interactive)", + "judge": "Model-based: In the wild", + "protocol": "1. Given a goal and a trajectory produced by an agent, an LLM judge classifies whether it successfully achieved 2. the goal. We use LLM judges relying on both the screenshot and UI tree.", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Crowdsourced annotations with expert review, hierarchical consistency checks, multi-round verification", - "baseline_models": "PSPNet: 43.29 mIoU DeepLab v3: 45.65 mIoU UPerNet: 44.85 mIoU SegFormer-B5: 51.8 mIoU", - "robustness_measures": "Inter-rater reliability; Multiple runs per sample", - "known_limitations": "Long-tail distribution of object classes; Annotation granularity varies; Some scenes have ambiguous boundaries; Challenging for rare categories", - "benchmarks_list": "Cityscapes, Pascal Context, COCO-Stuff, Mapillary Vistas" - } + "heldout_details": "", + "alignment_validation": "human operator is asked to judge whether the trajectory successfully achieves the goal or not. Then we assess the alignment between human judge and LLM judge to ensure a high alignment", + "baseline_models": "", + "robustness_measures": "Prompt variations tested; Multiple runs per sample; Temperature sensitivity tested; Repeated evaluations; Ablation studies; Inter-rater reliability; Significance testing", + "known_limitations": "-live and dynamic environment introduces uncontrollable factors (feature deprecation, version changes etc) that makes certain tasks unavailable after a period -sizable efforts required to manually set up prework in the environment ", + "benchmarks_list": "AndroidControl, Android in the wild" + }, + "evaluation_timestamp": "2024-08-06T01:09:12.351Z" }, { - "evaluation_name": "Mapillary Vistas", + "evaluation_name": "IntPhys 2", "metric_config": { - "evaluation_description": "Mapillary Vistas Standard Accuracy", + "evaluation_description": "IntPhys 2 Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -84,38 +86,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9248861406860315, + "score": 0.49780176299227696, "details": { - "subtask_a": 0.9248861406860315, - "subtask_b": 0.9248861406860315 + "subtask_a": 0.49780176299227696, + "subtask_b": 0.49780176299227696 } }, "factsheet": { - "purpose": "Research; Development; Deployment", - "principles_tested": "Semantic segmentation, Panoptic segmentation, Scene understanding, Robust perception", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image)", - "output_modality": "Structured Data", - "input_source": "User-generated content", - "output_source": "Expert annotations", + "purpose": "Research", + "principles_tested": "Intuitive Physics", + "functional_props": "Core Performance", + "input_modality": "Video", + "output_modality": "Scores/Embeddings", + "input_source": "New dataset (released with eval)", + "output_source": "Simulation-based", "size": "Medium (1K - 100K samples)", - "splits": "Train (18K), Val (2K), Test (5K)", + "splits": "Debug Set: 60 videos for Model calibration Main Set: 1,012 videos as Main evaluation set Held-Out Set: 344 videos as Test set", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives street-level image 2. Model predicts per-pixel semantic labels (66 classes) 3. mIoU computed across all classes", + "protocol": "Feed a video to the model Ask the model using specific prompts wether the video is physically plausible or not Check if the model's answer match the ground truth label ", "model_access": "Outputs", "has_heldout": true, - "alignment_validation": "Expert annotators, multi-stage quality control, geographic diversity validation", - "baseline_models": "PSPNet: 42.7 mIoU DeepLab v3+: 45.8 mIoU HRNetV2: 50.3 mIoU", - "robustness_measures": "Geographic diversity tested; Weather variations included; Multiple runs per sample", - "known_limitations": "Varying image quality from crowdsourced data; Camera parameter diversity; Some regions overrepresented; Annotation inconsistencies across diverse scenes", - "benchmarks_list": "Cityscapes, BDD100K, IDD, WildDash" - } + "heldout_details": "Held-Out Set: 344 videos as Test set", + "alignment_validation": "We ran human baselines to ensure that this task is easy for humans. ", + "baseline_models": "- Human baseline: 96% - Best model baseline: V-JEPA2 56%", + "robustness_measures": "Prompt variations tested; Multiple runs per sample; Temperature sensitivity tested", + "known_limitations": "Model can be very sensitive to the way they are prompted. Also the compression artefact of the video can also impact the results. Those are mostly limitations on the model sides, since human are not sensitive to those. ", + "benchmarks_list": "IntPhys: A Framework and Benchmark for Visual Intuitive Physics Reasoning, Riochet et al. 2020" + }, + "evaluation_timestamp": "2024-08-06T21:43:09.262Z" }, { - "evaluation_name": "Visual Genome", + "evaluation_name": "ImageNet", "metric_config": { - "evaluation_description": "Visual Genome Standard Accuracy", + "evaluation_description": "ImageNet Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -123,38 +127,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8560665838440542, + "score": 0.9104325906929738, "details": { - "subtask_a": 0.8560665838440542, - "subtask_b": 0.8560665838440542 + "subtask_a": 0.9104325906929738, + "subtask_b": 0.9104325906929738 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Scene graph generation, Visual relationship detection, Visual question answering, Dense captioning", + "principles_tested": "Object recognition, Visual categorization, Multi-class classification", "functional_props": "Core Performance", "input_modality": "Vision (Image)", - "output_modality": "Structured Data; Text", - "input_source": "MS COCO", - "output_source": "Crowdsourced annotations", - "size": "Large (100K - 1M samples)", - "splits": "Train, Val, Test", + "output_modality": "Structured Data", + "input_source": "New dataset (released with eval)", + "output_source": "Human annotations", + "size": "Huge (> 10M samples)", + "splits": "Train, Validation, Test", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives image 2. Model predicts objects, attributes, and relationships 3. Recall@K metrics for scene graph components", + "protocol": "1. Model receives an image as input 2. Model predicts class label from 1000 classes 3. Top-1 and Top-5 accuracy computed against ground truth", "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Multi-round crowdsourced annotations with validation, consistency checks for relationships", - "baseline_models": "IMP: 14.6 R@50 (scene graph detection) Motifs: 21.4 R@50 VCTree: 22.0 R@50 GPS-Net: 24.0 R@50", - "robustness_measures": "Inter-rater reliability; Multiple runs per sample", - "known_limitations": "Long-tail distribution of relationships; Annotation inconsistencies; Subjective relationship definitions; Incomplete annotations (not all relationships captured)", - "benchmarks_list": "GQA, Scene Graph Benchmark, VRD, OpenImages V6" - } + "has_heldout": true, + "heldout_details": "Private test set maintained by organizers, used for annual ILSVRC competitions", + "alignment_validation": "Expert curation of hierarchical categories, manual verification of labels, consistency checks across similar categories", + "baseline_models": "AlexNet: 63.3% top-1 VGG: 71.5% top-1 ResNet-50: 76.1% top-1 ResNet-152: 78.3% top-1 EfficientNet-B7: 84.3% top-1 Human performance: ~95% top-5", + "robustness_measures": "Multiple runs per sample; Significance testing; Inter-rater reliability", + "known_limitations": "Class imbalance in some categories; Label noise in training set; Some ambiguous images with multiple valid labels; Bias towards certain object viewpoints and contexts", + "benchmarks_list": "ImageNet-V2, ImageNet-C, ImageNet-R, ImageNet-A, ImageNet-Sketch" + }, + "evaluation_timestamp": "2024-07-14T15:13:46.806Z" }, { - "evaluation_name": "DAVIS (Densely Annotated VIdeo Segmentation)", + "evaluation_name": "COCO (Common Objects in Context)", "metric_config": { - "evaluation_description": "DAVIS (Densely Annotated VIdeo Segmentation) Standard Accuracy", + "evaluation_description": "COCO (Common Objects in Context) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -162,38 +168,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9999401368162061, + "score": 0.40582171989763327, "details": { - "subtask_a": 0.9999401368162061, - "subtask_b": 0.9999401368162061 + "subtask_a": 0.40582171989763327, + "subtask_b": 0.40582171989763327 } }, "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Video object segmentation, Temporal consistency, Object tracking, Dense prediction", - "functional_props": "Core Performance; Robustness", - "input_modality": "Video", - "output_modality": "Structured Data", + "purpose": "Research; Development; Selection", + "principles_tested": "Object detection, Instance segmentation, Keypoint detection, Panoptic segmentation, Image captioning", + "functional_props": "Core Performance", + "input_modality": "Vision (Image)", + "output_modality": "Text; Structured Data", "input_source": "New dataset (released with eval)", "output_source": "Human annotations", - "size": "Small (< 1K samples)", - "splits": "Train/Val (60 sequences), Test-dev (30 sequences)", + "size": "Large (100K - 1M samples)", + "splits": "Train, Validation, Test", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives video sequence with first frame annotation 2. Model propagates segmentation to subsequent frames 3. J&F metric (region similarity and contour accuracy)", + "protocol": "1. Model receives image as input 2. Model outputs bounding boxes, segmentation masks, or captions 3. Metrics computed: mAP for detection, IoU for segmentation, BLEU/CIDEr for captioning", "model_access": "Outputs", "has_heldout": true, - "alignment_validation": "High-quality manual annotations with temporal consistency verification", - "baseline_models": "OSVOS: 79.8 J&F STM: 84.3 J&F XMem: 86.2 J&F", - "robustness_measures": "Temporal consistency tested; Occlusion robustness; Multiple runs per sample", - "known_limitations": "Small dataset size; Limited object categories; Short video sequences; Primarily objects with clear boundaries", - "benchmarks_list": "YouTube-VOS, FBMS, SegTrack, OVIS" - } + "heldout_details": "Test-dev and test-challenge splits maintained privately, submissions via evaluation server", + "alignment_validation": "Multi-annotator consensus for instance annotations, quality control through redundant labeling", + "baseline_models": "Faster R-CNN: 42.0 mAP Mask R-CNN: 37.1 mask mAP YOLOv8: 53.9 mAP Human performance (detection): ~70 mAP", + "robustness_measures": "Inter-rater reliability; Multiple runs per sample; Confidence intervals", + "known_limitations": "Small object detection remains challenging; Occlusion handling difficulties; Dataset bias toward certain object contexts; Annotation inconsistencies in crowded scenes", + "benchmarks_list": "LVIS, Objects365, Open Images, Visual Genome" + }, + "evaluation_timestamp": "2024-07-15T13:13:06.081Z" }, { - "evaluation_name": "BBH (BIG-Bench Hard)", + "evaluation_name": "CIFAR-10 and CIFAR-100", "metric_config": { - "evaluation_description": "BBH (BIG-Bench Hard) Standard Accuracy", + "evaluation_description": "CIFAR-10 and CIFAR-100 Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -201,38 +209,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9767719353008874, + "score": 0.5280190557363246, "details": { - "subtask_a": 0.9767719353008874, - "subtask_b": 0.9767719353008874 + "subtask_a": 0.5280190557363246, + "subtask_b": 0.5280190557363246 } }, "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Complex reasoning, Multi-step thinking, Challenging cognitive tasks, Chain-of-thought reasoning", - "functional_props": "Core Performance", - "input_modality": "Text", - "output_modality": "Text", - "input_source": "One or Multiple existing datasets", - "output_source": "Multiple/Mixed sources", + "purpose": "Research; Development; Selection", + "principles_tested": "Image classification, Transfer learning, Generalization", + "functional_props": "Core Performance; Robustness", + "input_modality": "Vision (Image)", + "output_modality": "Structured Data", + "input_source": "New dataset (released with eval)", + "output_source": "Human annotations", "size": "Medium (1K - 100K samples)", - "splits": "Test (6,511 examples across 23 tasks)", + "splits": "Train (50K), Test (10K)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives task from one of 23 challenging categories 2. Model generates answer 3. Performance aggregated across all tasks 4. Chain-of-thought prompting typically used", + "protocol": "1. Model receives 32x32 RGB image 2. Model predicts class label (10 or 100 classes) 3. Classification accuracy computed", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Tasks selected where models underperform humans, difficulty validated through empirical testing", - "baseline_models": "PaLM (540B): 56.5% average PaLM (540B) + CoT: 78.1% GPT-4: ~91% Human raters: ~92% average", - "robustness_measures": "Multiple tasks; Chain-of-thought evaluation; Human baseline comparison; Cross-model validation", - "known_limitations": "Only 23 tasks (limited coverage); Chain-of-thought prompting required for good performance; Task aggregation methodology; Rapidly saturating with newer models", - "benchmarks_list": "BIG-Bench, MMLU, AGIEval, HELM" - } + "heldout_details": "", + "alignment_validation": "Systematic sampling from larger dataset (80 million tiny images), human verification of labels", + "baseline_models": "CIFAR-10: ResNet-56: 93.03% Wide ResNet-28-10: 96.11% PyramidNet: 96.54% CIFAR-100: ResNet-56: 71.35% Wide ResNet-28-10: 81.15% PyramidNet: 83.78%", + "robustness_measures": "Multiple runs per sample; Seed variation tested; Ablation studies", + "known_limitations": "Low resolution (32x32) limits fine-grained recognition; Some label noise in CIFAR-100; Dataset size enables memorization in large models; Limited diversity in poses and contexts", + "benchmarks_list": "CIFAR-10-C, CIFAR-10.1, CIFAR-100-C, STL-10, Tiny ImageNet" + }, + "evaluation_timestamp": "2024-07-31T15:53:14.267Z" }, { - "evaluation_name": "TruthfulQA", + "evaluation_name": "Pascal VOC", "metric_config": { - "evaluation_description": "TruthfulQA Standard Accuracy", + "evaluation_description": "Pascal VOC Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -240,38 +250,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9502647129813138, + "score": 0.5854518234873589, "details": { - "subtask_a": 0.9502647129813138, - "subtask_b": 0.9502647129813138 + "subtask_a": 0.5854518234873589, + "subtask_b": 0.5854518234873589 } }, "factsheet": { - "purpose": "Research; Development; Safety", - "principles_tested": "Truthfulness, Factual accuracy, Resistance to misconceptions, Calibration", - "functional_props": "Safety; Core Performance; Calibration", - "input_modality": "Text", - "output_modality": "Text", + "purpose": "Research; Development; Selection", + "principles_tested": "Object detection, Semantic segmentation, Instance segmentation, Action classification", + "functional_props": "Core Performance", + "input_modality": "Vision (Image)", + "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Expert annotations", - "size": "Small (< 1K samples)", - "splits": "Test (817 questions across 38 categories)", + "output_source": "Human annotations", + "size": "Medium (1K - 100K samples)", + "splits": "Train, Validation, Test", "design": "Fixed data-driven (static test set)", - "judge": "Model-based: Expert; Human: Experts", - "protocol": "1. Model receives question designed to elicit false beliefs 2. Model generates answer 3. Answers judged for truthfulness and informativeness using GPT-judge or human evaluation", + "judge": "Automatic (Reference-based)", + "protocol": "1. Model receives image as input 2. Model predicts bounding boxes and class labels 3. mAP computed at IoU threshold 0.5", "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Expert-curated questions targeting known misconceptions, multi-rater validation of truth labels", - "baseline_models": "GPT-3 (175B): 58.0% GPT-3.5: 47.0% GPT-4: 59.0% Claude 2: 62.0% Human baseline: 94.0%", - "robustness_measures": "Multiple evaluation methods (MC1, MC2, generative); Human validation; Model-based evaluation correlation", - "known_limitations": "Subjective truthfulness judgments in some cases; Cultural bias in what constitutes 'truth'; Model-based evaluation may not align with human judgment; Limited coverage of misconceptions", - "benchmarks_list": "FactScore, HaluEval, SelfCheckGPT, FELM" - } + "has_heldout": true, + "heldout_details": "Test set labels held privately, evaluation via submission to organizers (historically)", + "alignment_validation": "Careful manual annotation with quality control, multiple annotators for difficult cases", + "baseline_models": "R-CNN: 58.5 mAP (VOC 2007) Fast R-CNN: 70.0 mAP Faster R-CNN: 75.9 mAP YOLOv3: 78.6 mAP", + "robustness_measures": "Inter-rater reliability; Significance testing", + "known_limitations": "Limited to 20 classes; Small dataset size by modern standards; Some annotation inconsistencies; Relatively simple backgrounds", + "benchmarks_list": "COCO, LVIS, Open Images, Cityscapes" + }, + "evaluation_timestamp": "2024-07-21T01:20:51.546Z" }, { - "evaluation_name": "KITTI", + "evaluation_name": "Cityscapes", "metric_config": { - "evaluation_description": "KITTI Standard Accuracy", + "evaluation_description": "Cityscapes Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -279,38 +291,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8766055643660604, + "score": 0.8433612345529187, "details": { - "subtask_a": 0.8766055643660604, - "subtask_b": 0.8766055643660604 + "subtask_a": 0.8433612345529187, + "subtask_b": 0.8433612345529187 } }, "factsheet": { "purpose": "Research; Development; Deployment", - "principles_tested": "3D object detection, Stereo vision, Optical flow, Visual odometry, Tracking", + "principles_tested": "Semantic segmentation, Instance segmentation, Scene understanding, Autonomous driving perception", "functional_props": "Core Performance; Robustness", "input_modality": "Vision (Image); Video", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Human annotations; Programmatically generated", + "output_source": "Expert annotations", "size": "Medium (1K - 100K samples)", - "splits": "Train, Val, Test", + "splits": "Train (2975), Val (500), Test (1525)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives stereo image pair or point cloud 2. Model predicts 3D bounding boxes, orientation, class 3. 3D AP computed at different difficulty levels (easy/moderate/hard)", + "protocol": "1. Model receives street scene image (2048×1024) 2. Model predicts per-pixel semantic class (19 or 30 classes) 3. IoU and accuracy metrics computed", "model_access": "Outputs", "has_heldout": true, - "alignment_validation": "LiDAR ground truth for 3D positions, manual verification of annotations, multi-sensor fusion for accuracy", - "baseline_models": "PointPillars: 79.05 AP (Car, Moderate) PV-RCNN: 83.90 AP (Car, Moderate) CenterPoint: 85.15 AP (Car, Moderate)", - "robustness_measures": "Multiple difficulty levels; Ablation studies; Significance testing", - "known_limitations": "Limited to specific geographic region; Weather bias (mostly clear); Limited nighttime data; Class imbalance (cars dominate)", - "benchmarks_list": "nuScenes, Waymo Open Dataset, Argoverse, A2D2, Lyft Level 5" - } + "heldout_details": "Test set labels private, evaluation via online server with leaderboard", + "alignment_validation": "Expert annotators with domain knowledge, multi-pass quality control, consistency verification across video sequences", + "baseline_models": "FCN-8s: 65.3 mIoU DeepLab v3+: 82.1 mIoU HRNetV2: 83.0 mIoU SegFormer: 84.0 mIoU", + "robustness_measures": "Multiple runs per sample; Ablation studies; Significance testing", + "known_limitations": "Limited to European cities; Weather bias (mostly good conditions); Class imbalance for rare objects; Fine annotation boundaries challenging", + "benchmarks_list": "ADE20K, KITTI, Mapillary Vistas, BDD100K, nuScenes" + }, + "evaluation_timestamp": "2024-08-07T17:49:22.178Z" }, { - "evaluation_name": "UCF-101", + "evaluation_name": "ADE20K", "metric_config": { - "evaluation_description": "UCF-101 Standard Accuracy", + "evaluation_description": "ADE20K Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -318,38 +332,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8056383287007993, + "score": 0.7385681900404655, "details": { - "subtask_a": 0.8056383287007993, - "subtask_b": 0.8056383287007993 + "subtask_a": 0.7385681900404655, + "subtask_b": 0.7385681900404655 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Action recognition, Video classification, Temporal understanding", + "principles_tested": "Scene parsing, Semantic segmentation, Multi-scale recognition, Part segmentation", "functional_props": "Core Performance", - "input_modality": "Video", + "input_modality": "Vision (Image)", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", "output_source": "Human annotations", "size": "Medium (1K - 100K samples)", - "splits": "Train/Test (3 splits provided)", + "splits": "Train (20K), Val (2K), Test (3K)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives video clip 2. Model predicts action class (101 classes) 3. Average accuracy across 3 splits reported", + "protocol": "1. Model receives diverse scene image 2. Model predicts per-pixel semantic labels (150 classes) 3. Mean IoU and pixel accuracy computed", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Manual verification of action labels, removal of ambiguous videos", - "baseline_models": "Two-Stream CNN: 88.0% I3D: 95.6% SlowFast: 96.8% VideoMAE: 97.2%", - "robustness_measures": "Multiple evaluation splits; Seed variation tested", - "known_limitations": "YouTube videos may become unavailable; Camera motion and quality vary; Some action classes are very similar; Dataset saturation with modern methods", - "benchmarks_list": "HMDB-51, Kinetics, ActivityNet, Something-Something-V2" - } + "heldout_details": "", + "alignment_validation": "Crowdsourced annotations with expert review, hierarchical consistency checks, multi-round verification", + "baseline_models": "PSPNet: 43.29 mIoU DeepLab v3: 45.65 mIoU UPerNet: 44.85 mIoU SegFormer-B5: 51.8 mIoU", + "robustness_measures": "Inter-rater reliability; Multiple runs per sample", + "known_limitations": "Long-tail distribution of object classes; Annotation granularity varies; Some scenes have ambiguous boundaries; Challenging for rare categories", + "benchmarks_list": "Cityscapes, Pascal Context, COCO-Stuff, Mapillary Vistas" + }, + "evaluation_timestamp": "2024-08-08T08:57:25.622Z" }, { - "evaluation_name": "Waymo Open Dataset", + "evaluation_name": "Kinetics", "metric_config": { - "evaluation_description": "Waymo Open Dataset Standard Accuracy", + "evaluation_description": "Kinetics Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -357,38 +373,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9726918392065588, + "score": 0.47872691208723506, "details": { - "subtask_a": 0.9726918392065588, - "subtask_b": 0.9726918392065588 + "subtask_a": 0.47872691208723506, + "subtask_b": 0.47872691208723506 } }, "factsheet": { - "purpose": "Research; Development; Deployment", - "principles_tested": "3D object detection, 2D object detection, Tracking, Domain adaptation, Sensor fusion", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image); Video; Structured Data", + "purpose": "Research; Development", + "principles_tested": "Action recognition, Video understanding, Temporal reasoning, Human activity recognition", + "functional_props": "Core Performance", + "input_modality": "Video", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Human annotations; Programmatically generated", - "size": "Huge (> 10M samples)", - "splits": "Train (798), Val (202), Test (150)", + "output_source": "Human annotations", + "size": "Large (100K - 1M samples)", + "splits": "Train, Val, Test", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives multimodal sensor data 2. Model predicts 3D bounding boxes with tracking IDs 3. AP/APH metrics at different IoU thresholds and difficulty levels", + "protocol": "1. Model receives 10-second video clip 2. Model predicts action class (400/600/700 classes) 3. Top-1 and Top-5 accuracy computed", "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Multi-stage annotation pipeline with quality checks, temporal consistency validation", - "baseline_models": "PointPillars: 63.8 L2 APH (Vehicle) CenterPoint: 73.9 L2 APH PV-RCNN++: 77.8 L2 APH", - "robustness_measures": "Geographic diversity; Time of day variations; Weather conditions; Multiple difficulty levels", - "known_limitations": "Geographic concentration in specific cities; Sensor-specific challenges; Annotation latency for distant objects; Class imbalance", - "benchmarks_list": "nuScenes, KITTI, Argoverse 2, Once, Lyft Level 5" - } + "has_heldout": false, + "heldout_details": "", + "alignment_validation": "Human verification of video labels, removal of ambiguous clips, consistency checks for similar actions", + "baseline_models": "I3D: 71.1% top-1 (K400) SlowFast: 79.8% top-1 (K400) X3D: 80.4% top-1 (K400) VideoMAE: 81.5% top-1 (K400)", + "robustness_measures": "Multiple runs per sample; Temporal ordering tested", + "known_limitations": "YouTube videos may become unavailable over time; Some action classes overlap or are ambiguous; Camera viewpoint bias; Dataset drift as internet content changes", + "benchmarks_list": "UCF-101, HMDB-51, ActivityNet, Something-Something, Moments in Time" + }, + "evaluation_timestamp": "2024-07-23T05:35:32.393Z" }, { - "evaluation_name": "CelebA", + "evaluation_name": "KITTI", "metric_config": { - "evaluation_description": "CelebA Standard Accuracy", + "evaluation_description": "KITTI Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -396,38 +414,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8122873743552287, + "score": 0.3476914143454747, "details": { - "subtask_a": 0.8122873743552287, - "subtask_b": 0.8122873743552287 + "subtask_a": 0.3476914143454747, + "subtask_b": 0.3476914143454747 } }, "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Face attribute recognition, Face detection, Facial landmark detection, Identity recognition", - "functional_props": "Core Performance; Fairness", - "input_modality": "Vision (Image)", + "purpose": "Research; Development; Deployment", + "principles_tested": "3D object detection, Stereo vision, Optical flow, Visual odometry, Tracking", + "functional_props": "Core Performance; Robustness", + "input_modality": "Vision (Image); Video", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", - "size": "Large (100K - 1M samples)", - "splits": "Train (162,770), Val (19,867), Test (19,962)", + "output_source": "Human annotations; Programmatically generated", + "size": "Medium (1K - 100K samples)", + "splits": "Train, Val, Test", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives face image 2. Model predicts binary attributes (40 dimensions) 3. Per-attribute accuracy and mean accuracy computed", + "protocol": "1. Model receives stereo image pair or point cloud 2. Model predicts 3D bounding boxes, orientation, class 3. 3D AP computed at different difficulty levels (easy/moderate/hard)", "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Manual annotation with quality control, consistency verification across multiple attributes", - "baseline_models": "LNets+ANet: 87.30% mean accuracy Walk and Learn: 88.06% FaceNet: 89.35%", - "robustness_measures": "Inter-rater reliability; Multiple runs per sample", - "known_limitations": "Demographic bias (celebrity images); Some attributes are subjective; Label noise in some attributes; Privacy concerns with celebrity images; Lighting and pose variation", - "benchmarks_list": "LFW, VGGFace2, MS-Celeb-1M, FFHQ" - } + "has_heldout": true, + "heldout_details": "Test set labels held privately, evaluation via online server with public leaderboard", + "alignment_validation": "LiDAR ground truth for 3D positions, manual verification of annotations, multi-sensor fusion for accuracy", + "baseline_models": "PointPillars: 79.05 AP (Car, Moderate) PV-RCNN: 83.90 AP (Car, Moderate) CenterPoint: 85.15 AP (Car, Moderate)", + "robustness_measures": "Multiple difficulty levels; Ablation studies; Significance testing", + "known_limitations": "Limited to specific geographic region; Weather bias (mostly clear); Limited nighttime data; Class imbalance (cars dominate)", + "benchmarks_list": "nuScenes, Waymo Open Dataset, Argoverse, A2D2, Lyft Level 5" + }, + "evaluation_timestamp": "2024-07-13T02:41:45.437Z" }, { - "evaluation_name": "HellaSwag", + "evaluation_name": "Places365", "metric_config": { - "evaluation_description": "HellaSwag Standard Accuracy", + "evaluation_description": "Places365 Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -435,38 +455,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.886277502483783, + "score": 0.8807403220941152, "details": { - "subtask_a": 0.886277502483783, - "subtask_b": 0.886277502483783 + "subtask_a": 0.8807403220941152, + "subtask_b": 0.8807403220941152 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Commonsense reasoning, Physical understanding, Situation modeling, Plausibility judgment", - "functional_props": "Core Performance; Robustness", - "input_modality": "Text", - "output_modality": "Text", + "principles_tested": "Scene recognition, Scene classification, Environmental understanding, Context recognition", + "functional_props": "Core Performance", + "input_modality": "Vision (Image)", + "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Crowdsourced annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (39,905), Val (10,042), Test (10,003)", + "output_source": "Human annotations", + "size": "Very Huge (> 100M samples)", + "splits": "Train, Val, Test", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives scenario context 2. Model selects most plausible continuation from 4 options 3. Accuracy computed", + "protocol": "1. Model receives scene image 2. Model predicts scene category (365 classes) 3. Top-1 and Top-5 accuracy computed", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Adversarial filtering using BERT to ensure difficulty, human validation of plausibility", - "baseline_models": "BERT-Large: 47.3% GPT-2: 50.9% GPT-3: 78.9% GPT-4: 95.3% Human performance: 95.6%", - "robustness_measures": "Adversarial filtering; Multiple runs per sample; Human baseline comparison", - "known_limitations": "Dataset may be easier than originally intended for modern LLMs; Multiple choice format; Adversarial examples may have artifacts; Potential data contamination", - "benchmarks_list": "PIQA, WinoGrande, CommonsenseQA, ARC" - } + "heldout_details": "", + "alignment_validation": "Human verification, consistency checks for scene categories, hierarchical taxonomy validation", + "baseline_models": "ResNet-152: 55.24% top-1 DenseNet-161: 56.12% top-1 ResNeXt-101: 56.05% top-1", + "robustness_measures": "Inter-rater reliability; Multiple runs per sample", + "known_limitations": "Some scene categories overlap semantically; Cultural bias in scene definitions; Indoor scenes better represented than outdoor; Ambiguous boundary cases", + "benchmarks_list": "SUN397, MIT Indoor 67, Scene-15, ADE20K" + }, + "evaluation_timestamp": "2024-07-25T16:03:28.578Z" }, { - "evaluation_name": "Fashion-MNIST", + "evaluation_name": "UCF-101", "metric_config": { - "evaluation_description": "Fashion-MNIST Standard Accuracy", + "evaluation_description": "UCF-101 Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -474,38 +496,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9397737938198042, + "score": 0.8424044957403503, "details": { - "subtask_a": 0.9397737938198042, - "subtask_b": 0.9397737938198042 + "subtask_a": 0.8424044957403503, + "subtask_b": 0.8424044957403503 } }, "factsheet": { - "purpose": "Research; Development; Selection", - "principles_tested": "Image classification, Transfer learning, Benchmark comparison", + "purpose": "Research; Development", + "principles_tested": "Action recognition, Video classification, Temporal understanding", "functional_props": "Core Performance", - "input_modality": "Vision (Image)", + "input_modality": "Video", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", "output_source": "Human annotations", "size": "Medium (1K - 100K samples)", - "splits": "Train (60K), Test (10K)", + "splits": "Train/Test (3 splits provided)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives 28x28 grayscale image 2. Model predicts fashion category (10 classes) 3. Classification accuracy computed", + "protocol": "1. Model receives video clip 2. Model predicts action class (101 classes) 3. Average accuracy across 3 splits reported", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Product catalog labels verified by domain experts", - "baseline_models": "Linear Classifier: 83.7% CNN: 93.5% ResNet: 94.9% Vision Transformer: 95.1%", - "robustness_measures": "Multiple runs per sample; Seed variation tested", - "known_limitations": "Low resolution (28x28); Grayscale only; Limited intra-class variation; Some categories overlap visually", - "benchmarks_list": "MNIST, EMNIST, Kuzushiji-MNIST, DeepFashion" - } + "heldout_details": "", + "alignment_validation": "Manual verification of action labels, removal of ambiguous videos", + "baseline_models": "Two-Stream CNN: 88.0% I3D: 95.6% SlowFast: 96.8% VideoMAE: 97.2%", + "robustness_measures": "Multiple evaluation splits; Seed variation tested", + "known_limitations": "YouTube videos may become unavailable; Camera motion and quality vary; Some action classes are very similar; Dataset saturation with modern methods", + "benchmarks_list": "HMDB-51, Kinetics, ActivityNet, Something-Something-V2" + }, + "evaluation_timestamp": "2024-08-10T06:46:43.976Z" }, { - "evaluation_name": "Cityscapes", + "evaluation_name": "NYU Depth V2", "metric_config": { - "evaluation_description": "Cityscapes Standard Accuracy", + "evaluation_description": "NYU Depth V2 Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -513,38 +537,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8112883764278835, + "score": 0.5960847112010853, "details": { - "subtask_a": 0.8112883764278835, - "subtask_b": 0.8112883764278835 + "subtask_a": 0.5960847112010853, + "subtask_b": 0.5960847112010853 } }, "factsheet": { - "purpose": "Research; Development; Deployment", - "principles_tested": "Semantic segmentation, Instance segmentation, Scene understanding, Autonomous driving perception", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image); Video", + "purpose": "Research; Development", + "principles_tested": "Depth estimation, RGB-D understanding, Indoor scene parsing, 3D reconstruction", + "functional_props": "Core Performance", + "input_modality": "Vision (Image); Structured Data", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", "output_source": "Expert annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (2975), Val (500), Test (1525)", + "size": "Small (< 1K samples)", + "splits": "Train (795), Test (654)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives street scene image (2048×1024) 2. Model predicts per-pixel semantic class (19 or 30 classes) 3. IoU and accuracy metrics computed", + "protocol": "1. Model receives RGB image 2. Model predicts depth map or semantic segmentation 3. RMSE, absolute relative error, and accuracy metrics computed", "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Expert annotators with domain knowledge, multi-pass quality control, consistency verification across video sequences", - "baseline_models": "FCN-8s: 65.3 mIoU DeepLab v3+: 82.1 mIoU HRNetV2: 83.0 mIoU SegFormer: 84.0 mIoU", - "robustness_measures": "Multiple runs per sample; Ablation studies; Significance testing", - "known_limitations": "Limited to European cities; Weather bias (mostly good conditions); Class imbalance for rare objects; Fine annotation boundaries challenging", - "benchmarks_list": "ADE20K, KITTI, Mapillary Vistas, BDD100K, nuScenes" - } + "has_heldout": false, + "heldout_details": "", + "alignment_validation": "Kinect sensor ground truth with manual alignment corrections, multi-view consistency", + "baseline_models": "Depth Estimation: Eigen et al.: 0.641 RMSE AdaBins: 0.364 RMSE BTS: 0.392 RMSE", + "robustness_measures": "Multiple runs per sample; Ablation studies", + "known_limitations": "Small dataset size; Limited to indoor scenes; Kinect depth sensor limitations (range, IR interference); Mostly residential environments", + "benchmarks_list": "ScanNet, Matterport3D, SUNRGB-D, KITTI Depth" + }, + "evaluation_timestamp": "2024-07-15T00:47:51.047Z" }, { - "evaluation_name": "MPII Human Pose", + "evaluation_name": "CelebA", "metric_config": { - "evaluation_description": "MPII Human Pose Standard Accuracy", + "evaluation_description": "CelebA Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -552,38 +578,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9897615029006273, + "score": 0.7722569802453403, "details": { - "subtask_a": 0.9897615029006273, - "subtask_b": 0.9897615029006273 + "subtask_a": 0.7722569802453403, + "subtask_b": 0.7722569802453403 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Human pose estimation, Keypoint detection, Articulated pose estimation, Activity recognition", - "functional_props": "Core Performance", + "principles_tested": "Face attribute recognition, Face detection, Facial landmark detection, Identity recognition", + "functional_props": "Core Performance; Fairness", "input_modality": "Vision (Image)", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", "output_source": "Human annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (~29K people), Test (~12K people)", + "size": "Large (100K - 1M samples)", + "splits": "Train (162,770), Val (19,867), Test (19,962)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives image with person 2. Model predicts 2D joint locations (16 joints) 3. PCKh (Percentage of Correct Keypoints) at various thresholds", + "protocol": "1. Model receives face image 2. Model predicts binary attributes (40 dimensions) 3. Per-attribute accuracy and mean accuracy computed", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Manual annotation with consistency checks, multi-annotator agreement for difficult poses", - "baseline_models": "Hourglass Network: 90.9 PCKh@0.5 HRNet: 92.3 PCKh@0.5 SimpleBaseline: 91.5 PCKh@0.5", - "robustness_measures": "Multiple difficulty levels; Inter-rater reliability; Occlusion analysis", - "known_limitations": "2D annotations only (no 3D); Occlusion and truncation challenges; Some joint definitions ambiguous; Dataset bias toward certain activities", - "benchmarks_list": "COCO Keypoints, Human3.6M, PoseTrack, CrowdPose" - } + "heldout_details": "", + "alignment_validation": "Manual annotation with quality control, consistency verification across multiple attributes", + "baseline_models": "LNets+ANet: 87.30% mean accuracy Walk and Learn: 88.06% FaceNet: 89.35%", + "robustness_measures": "Inter-rater reliability; Multiple runs per sample", + "known_limitations": "Demographic bias (celebrity images); Some attributes are subjective; Label noise in some attributes; Privacy concerns with celebrity images; Lighting and pose variation", + "benchmarks_list": "LFW, VGGFace2, MS-Celeb-1M, FFHQ" + }, + "evaluation_timestamp": "2024-07-12T11:18:59.718Z" }, { - "evaluation_name": "WinoGrande", + "evaluation_name": "Visual Genome", "metric_config": { - "evaluation_description": "WinoGrande Standard Accuracy", + "evaluation_description": "Visual Genome Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -591,38 +619,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9478475154992905, + "score": 0.3980771041596804, "details": { - "subtask_a": 0.9478475154992905, - "subtask_b": 0.9478475154992905 + "subtask_a": 0.3980771041596804, + "subtask_b": 0.3980771041596804 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Commonsense reasoning, Coreference resolution, World knowledge, Causal reasoning", - "functional_props": "Core Performance; Robustness", - "input_modality": "Text", - "output_modality": "Text", - "input_source": "New dataset (released with eval)", + "principles_tested": "Scene graph generation, Visual relationship detection, Visual question answering, Dense captioning", + "functional_props": "Core Performance", + "input_modality": "Vision (Image)", + "output_modality": "Structured Data; Text", + "input_source": "MS COCO", "output_source": "Crowdsourced annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (40,398), Dev (1,267), Test (1,767)", + "size": "Large (100K - 1M samples)", + "splits": "Train, Val, Test", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives sentence with pronoun ambiguity 2. Model selects correct referent from 2 options 3. Accuracy computed", + "protocol": "1. Model receives image 2. Model predicts objects, attributes, and relationships 3. Recall@K metrics for scene graph components", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Adversarial filtering using language models, crowdsourced generation with validation", - "baseline_models": "BERT-Large: 59.4% RoBERTa-Large: 79.1% GPT-3: 70.2% GPT-4: 87.5% Human performance: 94.0%", - "robustness_measures": "Adversarial filtering; Large-scale dataset; Multiple difficulty levels", - "known_limitations": "Binary choice may be limiting; Adversarial filtering may introduce artifacts; Saturation with modern models; Limited reasoning depth", - "benchmarks_list": "Winograd Schema Challenge, COPA, CommonsenseQA, PIQA" - } + "heldout_details": "", + "alignment_validation": "Multi-round crowdsourced annotations with validation, consistency checks for relationships", + "baseline_models": "IMP: 14.6 R@50 (scene graph detection) Motifs: 21.4 R@50 VCTree: 22.0 R@50 GPS-Net: 24.0 R@50", + "robustness_measures": "Inter-rater reliability; Multiple runs per sample", + "known_limitations": "Long-tail distribution of relationships; Annotation inconsistencies; Subjective relationship definitions; Incomplete annotations (not all relationships captured)", + "benchmarks_list": "GQA, Scene Graph Benchmark, VRD, OpenImages V6" + }, + "evaluation_timestamp": "2024-08-10T12:51:32.233Z" }, { - "evaluation_name": "DROP (Discrete Reasoning Over Paragraphs)", + "evaluation_name": "LVIS (Large Vocabulary Instance Segmentation)", "metric_config": { - "evaluation_description": "DROP (Discrete Reasoning Over Paragraphs) Standard Accuracy", + "evaluation_description": "LVIS (Large Vocabulary Instance Segmentation) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -630,38 +660,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8524184477297209, + "score": 0.6234930539489729, "details": { - "subtask_a": 0.8524184477297209, - "subtask_b": 0.8524184477297209 + "subtask_a": 0.6234930539489729, + "subtask_b": 0.6234930539489729 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Reading comprehension, Numerical reasoning, Discrete operations, Multi-hop reasoning", - "functional_props": "Core Performance", - "input_modality": "Text", - "output_modality": "Text", - "input_source": "New dataset (released with eval)", - "output_source": "Crowdsourced annotations", + "principles_tested": "Instance segmentation, Long-tail recognition, Object detection, Fine-grained categorization", + "functional_props": "Core Performance; Robustness", + "input_modality": "Vision (Image)", + "output_modality": "Structured Data", + "input_source": "MS COCO", + "output_source": "Expert annotations", "size": "Large (100K - 1M samples)", - "splits": "Train (77,409), Dev (9,536), Test (9,622)", + "splits": "Train, Val, Test", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives paragraph and question 2. Model generates answer (number, span, or date) 3. F1 and Exact Match metrics computed", + "protocol": "1. Model receives image 2. Model predicts instance masks and categories (1203 classes) 3. AP computed separately for rare, common, and frequent categories", "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Crowdsourced questions with verification, requires discrete reasoning operations confirmed through analysis", - "baseline_models": "BERT: 47.0 F1 RoBERTa: 80.9 F1 GPT-3: 29.0 F1 GPT-4: 80.9 F1 Human performance: 96.4 F1", - "robustness_measures": "Multiple answer types; Question type analysis; Human baseline comparison", - "known_limitations": "Requires careful answer extraction; Some questions ambiguous; Numerical reasoning can be brittle; Limited diversity in reasoning types", - "benchmarks_list": "SQuAD, NewsQA, NaturalQuestions, NumGLUE, TAT-QA" - } + "has_heldout": true, + "heldout_details": "Test set labels private, evaluation via online server", + "alignment_validation": "Expert annotators with WordNet taxonomy, quality control for long-tail categories", + "baseline_models": "Mask R-CNN: 21.2 AP (v1.0) Cascade R-CNN: 26.2 AP Swin Transformer: 50.9 AP", + "robustness_measures": "Multiple runs per sample; Ablation studies; Category frequency stratification", + "known_limitations": "Rare categories have very few examples; Annotation cost for 1203 categories; Some category definitions overlap; Challenging for zero-shot generalization", + "benchmarks_list": "COCO, Objects365, OpenImages, iNaturalist" + }, + "evaluation_timestamp": "2024-07-14T17:59:07.085Z" }, { - "evaluation_name": "ActivityNet", + "evaluation_name": "Mapillary Vistas", "metric_config": { - "evaluation_description": "ActivityNet Standard Accuracy", + "evaluation_description": "Mapillary Vistas Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -669,38 +701,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9494105638093787, + "score": 0.5478391877447762, "details": { - "subtask_a": 0.9494105638093787, - "subtask_b": 0.9494105638093787 + "subtask_a": 0.5478391877447762, + "subtask_b": 0.5478391877447762 } }, "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Temporal action detection, Action recognition, Dense video captioning, Video understanding", - "functional_props": "Core Performance", - "input_modality": "Video", - "output_modality": "Structured Data; Text", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", + "purpose": "Research; Development; Deployment", + "principles_tested": "Semantic segmentation, Panoptic segmentation, Scene understanding, Robust perception", + "functional_props": "Core Performance; Robustness", + "input_modality": "Vision (Image)", + "output_modality": "Structured Data", + "input_source": "User-generated content", + "output_source": "Expert annotations", "size": "Medium (1K - 100K samples)", - "splits": "Train (50%), Val (25%), Test (25%)", + "splits": "Train (18K), Val (2K), Test (5K)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives untrimmed video 2. Model predicts action segments with class labels 3. mAP at different IoU thresholds (0.5, 0.75, 0.95)", + "protocol": "1. Model receives street-level image 2. Model predicts per-pixel semantic labels (66 classes) 3. mIoU computed across all classes", "model_access": "Outputs", "has_heldout": true, - "alignment_validation": "Multi-annotator temporal boundary agreement, consistency checks for activity definitions", - "baseline_models": "SSN: 41.3 mAP@0.5 BMN: 50.1 mAP@0.5 TALLFormer: 59.8 mAP@0.5", - "robustness_measures": "Inter-rater reliability; Multiple IoU thresholds; Temporal boundary sensitivity", - "known_limitations": "YouTube video availability issues; Temporal boundary ambiguity; Action class overlap; Video quality variance", - "benchmarks_list": "THUMOS14, Kinetics, Charades, MultiTHUMOS, AVA" - } + "heldout_details": "Test set labels private, evaluation via online platform", + "alignment_validation": "Expert annotators, multi-stage quality control, geographic diversity validation", + "baseline_models": "PSPNet: 42.7 mIoU DeepLab v3+: 45.8 mIoU HRNetV2: 50.3 mIoU", + "robustness_measures": "Geographic diversity tested; Weather variations included; Multiple runs per sample", + "known_limitations": "Varying image quality from crowdsourced data; Camera parameter diversity; Some regions overrepresented; Annotation inconsistencies across diverse scenes", + "benchmarks_list": "Cityscapes, BDD100K, IDD, WildDash" + }, + "evaluation_timestamp": "2024-07-23T12:22:18.653Z" }, { - "evaluation_name": "MMLU (Massive Multitask Language Understanding)", + "evaluation_name": "MPII Human Pose", "metric_config": { - "evaluation_description": "MMLU (Massive Multitask Language Understanding) Standard Accuracy", + "evaluation_description": "MPII Human Pose Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -708,38 +742,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9082783094368954, + "score": 0.33951331310169836, "details": { - "subtask_a": 0.9082783094368954, - "subtask_b": 0.9082783094368954 + "subtask_a": 0.33951331310169836, + "subtask_b": 0.33951331310169836 } }, "factsheet": { - "purpose": "Research; Development; Selection", - "principles_tested": "World knowledge, Reasoning, Domain expertise across 57 subjects", + "purpose": "Research; Development", + "principles_tested": "Human pose estimation, Keypoint detection, Articulated pose estimation, Activity recognition", "functional_props": "Core Performance", - "input_modality": "Text", - "output_modality": "Text", + "input_modality": "Vision (Image)", + "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Expert annotations", + "output_source": "Human annotations", "size": "Medium (1K - 100K samples)", - "splits": "Dev (5 shot examples per task), Test (285 questions per task average)", + "splits": "Train (~29K people), Test (~12K people)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives multiple choice question with 4 options 2. Model predicts answer (A/B/C/D) 3. Accuracy computed per subject and overall", + "protocol": "1. Model receives image with person 2. Model predicts 2D joint locations (16 joints) 3. PCKh (Percentage of Correct Keypoints) at various thresholds", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Questions sourced from practice exams and tests, expert review for correctness", - "baseline_models": "GPT-3 (175B): 43.9% GPT-3.5: 70.0% GPT-4: 86.4% Claude 3.5 Sonnet: 88.7% Random baseline: 25%", - "robustness_measures": "Multiple evaluation runs; Few-shot prompting variations; Subject-wise analysis", - "known_limitations": "Multiple choice format may not reflect real-world usage; Some questions have ambiguous answers; Dataset contamination concerns with web-trained models; Cultural and knowledge cutoff biases", - "benchmarks_list": "MMLU-Pro, AGIEval, C-Eval, CMMLU" - } + "heldout_details": "", + "alignment_validation": "Manual annotation with consistency checks, multi-annotator agreement for difficult poses", + "baseline_models": "Hourglass Network: 90.9 PCKh@0.5 HRNet: 92.3 PCKh@0.5 SimpleBaseline: 91.5 PCKh@0.5", + "robustness_measures": "Multiple difficulty levels; Inter-rater reliability; Occlusion analysis", + "known_limitations": "2D annotations only (no 3D); Occlusion and truncation challenges; Some joint definitions ambiguous; Dataset bias toward certain activities", + "benchmarks_list": "COCO Keypoints, Human3.6M, PoseTrack, CrowdPose" + }, + "evaluation_timestamp": "2024-07-15T13:51:48.600Z" }, { - "evaluation_name": "Kinetics", + "evaluation_name": "Open Images", "metric_config": { - "evaluation_description": "Kinetics Standard Accuracy", + "evaluation_description": "Open Images Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -747,38 +783,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8312419670034569, + "score": 0.785696265882706, "details": { - "subtask_a": 0.8312419670034569, - "subtask_b": 0.8312419670034569 + "subtask_a": 0.785696265882706, + "subtask_b": 0.785696265882706 } }, "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Action recognition, Video understanding, Temporal reasoning, Human activity recognition", - "functional_props": "Core Performance", - "input_modality": "Video", + "purpose": "Research; Development; Selection", + "principles_tested": "Object detection, Image classification, Visual relationship detection, Instance segmentation", + "functional_props": "Core Performance; Robustness", + "input_modality": "Vision (Image)", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", - "size": "Large (100K - 1M samples)", + "output_source": "Human annotations; Crowdsourced annotations", + "size": "Very Huge (> 100M samples)", "splits": "Train, Val, Test", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives 10-second video clip 2. Model predicts action class (400/600/700 classes) 3. Top-1 and Top-5 accuracy computed", + "protocol": "1. Model receives image 2. Model performs classification, detection, or segmentation 3. mAP computed for detection, top-k accuracy for classification", "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Human verification of video labels, removal of ambiguous clips, consistency checks for similar actions", - "baseline_models": "I3D: 71.1% top-1 (K400) SlowFast: 79.8% top-1 (K400) X3D: 80.4% top-1 (K400) VideoMAE: 81.5% top-1 (K400)", - "robustness_measures": "Multiple runs per sample; Temporal ordering tested", - "known_limitations": "YouTube videos may become unavailable over time; Some action classes overlap or are ambiguous; Camera viewpoint bias; Dataset drift as internet content changes", - "benchmarks_list": "UCF-101, HMDB-51, ActivityNet, Something-Something, Moments in Time" - } + "has_heldout": true, + "heldout_details": "Test set labels private for challenges, public validation set available", + "alignment_validation": "Multi-stage crowdsourced annotation with verification, automated quality filters", + "baseline_models": "Faster R-CNN: 54.3 mAP (detection) YOLOv4: 55.8 mAP EfficientDet: 56.1 mAP", + "robustness_measures": "Multiple runs per sample; Confidence intervals; Large-scale diversity", + "known_limitations": "Long-tail class distribution; Annotation inconsistencies at scale; Some classes poorly defined; Label noise in crowdsourced annotations", + "benchmarks_list": "COCO, LVIS, Objects365, ImageNet" + }, + "evaluation_timestamp": "2024-07-22T05:23:36.143Z" }, { - "evaluation_name": "BIG-bench (Beyond the Imitation Game)", + "evaluation_name": "ScanNet", "metric_config": { - "evaluation_description": "BIG-bench (Beyond the Imitation Game) Standard Accuracy", + "evaluation_description": "ScanNet Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -786,38 +824,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8642589329880018, + "score": 0.3942148543035357, "details": { - "subtask_a": 0.8642589329880018, - "subtask_b": 0.8642589329880018 + "subtask_a": 0.3942148543035357, + "subtask_b": 0.3942148543035357 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Diverse capabilities across 204 tasks including reasoning, knowledge, language understanding, bias detection", - "functional_props": "Core Performance; Fairness; Robustness", - "input_modality": "Text", - "output_modality": "Text", + "principles_tested": "3D scene understanding, Semantic segmentation, Instance segmentation, 3D reconstruction", + "functional_props": "Core Performance", + "input_modality": "Vision (Image); Video; Structured Data", + "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Multiple/Mixed sources", - "size": "Large (100K - 1M samples)", - "splits": "Varies by task", - "design": "Composite", - "judge": "Automatic (Reference-based); Model-based: In the wild", - "protocol": "1. Model evaluated on 204 diverse tasks 2. Each task has its own evaluation protocol 3. Performance aggregated across tasks", + "output_source": "Human annotations; Programmatically generated", + "size": "Medium (1K - 100K samples)", + "splits": "Train (1201), Val (312), Test (100)", + "design": "Fixed data-driven (static test set)", + "judge": "Automatic (Reference-based)", + "protocol": "1. Model receives 3D scene (RGB-D scans) 2. Model predicts semantic labels or instance masks 3. mIoU for semantic segmentation, mAP for instance segmentation", "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Crowdsourced task creation with quality review, diverse authorship for broad coverage", - "baseline_models": "Average human rater: 89.0% Few-shot PaLM (540B): 65.7% GPT-4: ~83% (estimated on BIG-Bench Hard)", - "robustness_measures": "Multiple tasks provide robustness; Human baseline comparison; Cross-task analysis", - "known_limitations": "Task quality varies; Some tasks too easy or too hard; Computational cost of running all 204 tasks; Aggregation methodology debatable", - "benchmarks_list": "BIG-Bench Hard, MMLU, HELM, SuperGLUE" - } + "has_heldout": true, + "heldout_details": "Test set labels private, evaluation via online benchmark server", + "alignment_validation": "Manual verification of 3D reconstructions, multi-annotator consistency for semantic labels", + "baseline_models": "PointNet++: 53.5 mIoU SparseConvNet: 72.5 mIoU MinkowskiNet: 73.6 mIoU", + "robustness_measures": "Multiple runs per sample; Ablation studies", + "known_limitations": "Limited to indoor scenes; Reconstruction artifacts; Scanning noise and occlusions; Limited scene diversity (mostly offices and apartments)", + "benchmarks_list": "Matterport3D, S3DIS, 2D-3D-S, ARKitScenes" + }, + "evaluation_timestamp": "2024-07-31T22:14:09.135Z" }, { - "evaluation_name": "Pascal VOC", + "evaluation_name": "nuScenes", "metric_config": { - "evaluation_description": "Pascal VOC Standard Accuracy", + "evaluation_description": "nuScenes Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -825,38 +865,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9307716089177187, + "score": 0.66479699085204, "details": { - "subtask_a": 0.9307716089177187, - "subtask_b": 0.9307716089177187 + "subtask_a": 0.66479699085204, + "subtask_b": 0.66479699085204 } }, "factsheet": { - "purpose": "Research; Development; Selection", - "principles_tested": "Object detection, Semantic segmentation, Instance segmentation, Action classification", - "functional_props": "Core Performance", - "input_modality": "Vision (Image)", + "purpose": "Research; Development; Deployment", + "principles_tested": "3D object detection, Multi-object tracking, Prediction, Sensor fusion, Scene understanding", + "functional_props": "Core Performance; Robustness", + "input_modality": "Vision (Image); Video; Structured Data", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", "output_source": "Human annotations", "size": "Medium (1K - 100K samples)", - "splits": "Train, Validation, Test", + "splits": "Train (700), Val (150), Test (150)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives image as input 2. Model predicts bounding boxes and class labels 3. mAP computed at IoU threshold 0.5", + "protocol": "1. Model receives multimodal sensor data (cameras, LiDAR, radar) 2. Model predicts 3D bounding boxes and tracking IDs 3. NDS (nuScenes Detection Score) and mAP computed", "model_access": "Outputs", "has_heldout": true, - "alignment_validation": "Careful manual annotation with quality control, multiple annotators for difficult cases", - "baseline_models": "R-CNN: 58.5 mAP (VOC 2007) Fast R-CNN: 70.0 mAP Faster R-CNN: 75.9 mAP YOLOv3: 78.6 mAP", - "robustness_measures": "Inter-rater reliability; Significance testing", - "known_limitations": "Limited to 20 classes; Small dataset size by modern standards; Some annotation inconsistencies; Relatively simple backgrounds", - "benchmarks_list": "COCO, LVIS, Open Images, Cityscapes" - } + "heldout_details": "Test set labels private, evaluation via online leaderboard", + "alignment_validation": "Expert annotators, multi-sensor consistency checks, temporal coherence validation", + "baseline_models": "PointPillars: 45.3 NDS CenterPoint: 65.5 NDS BEVFusion: 72.9 NDS", + "robustness_measures": "Geographic diversity; Weather conditions; Day/night variations; Multiple runs per sample", + "known_limitations": "Limited geographic coverage (Boston, Singapore); Sensor calibration challenges; Annotation latency for fast-moving objects; Class imbalance", + "benchmarks_list": "Waymo Open Dataset, KITTI, Argoverse, Lyft Level 5, A2D2" + }, + "evaluation_timestamp": "2024-07-21T23:35:21.751Z" }, { - "evaluation_name": "NYU Depth V2", + "evaluation_name": "ActivityNet", "metric_config": { - "evaluation_description": "NYU Depth V2 Standard Accuracy", + "evaluation_description": "ActivityNet Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -864,38 +906,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9422423186897226, + "score": 0.5758457068124234, "details": { - "subtask_a": 0.9422423186897226, - "subtask_b": 0.9422423186897226 + "subtask_a": 0.5758457068124234, + "subtask_b": 0.5758457068124234 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Depth estimation, RGB-D understanding, Indoor scene parsing, 3D reconstruction", + "principles_tested": "Temporal action detection, Action recognition, Dense video captioning, Video understanding", "functional_props": "Core Performance", - "input_modality": "Vision (Image); Structured Data", - "output_modality": "Structured Data", + "input_modality": "Video", + "output_modality": "Structured Data; Text", "input_source": "New dataset (released with eval)", - "output_source": "Expert annotations", - "size": "Small (< 1K samples)", - "splits": "Train (795), Test (654)", + "output_source": "Human annotations", + "size": "Medium (1K - 100K samples)", + "splits": "Train (50%), Val (25%), Test (25%)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives RGB image 2. Model predicts depth map or semantic segmentation 3. RMSE, absolute relative error, and accuracy metrics computed", + "protocol": "1. Model receives untrimmed video 2. Model predicts action segments with class labels 3. mAP at different IoU thresholds (0.5, 0.75, 0.95)", "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Kinect sensor ground truth with manual alignment corrections, multi-view consistency", - "baseline_models": "Depth Estimation: Eigen et al.: 0.641 RMSE AdaBins: 0.364 RMSE BTS: 0.392 RMSE", - "robustness_measures": "Multiple runs per sample; Ablation studies", - "known_limitations": "Small dataset size; Limited to indoor scenes; Kinect depth sensor limitations (range, IR interference); Mostly residential environments", - "benchmarks_list": "ScanNet, Matterport3D, SUNRGB-D, KITTI Depth" - } + "has_heldout": true, + "heldout_details": "Test set used for annual challenges, labels withheld", + "alignment_validation": "Multi-annotator temporal boundary agreement, consistency checks for activity definitions", + "baseline_models": "SSN: 41.3 mAP@0.5 BMN: 50.1 mAP@0.5 TALLFormer: 59.8 mAP@0.5", + "robustness_measures": "Inter-rater reliability; Multiple IoU thresholds; Temporal boundary sensitivity", + "known_limitations": "YouTube video availability issues; Temporal boundary ambiguity; Action class overlap; Video quality variance", + "benchmarks_list": "THUMOS14, Kinetics, Charades, MultiTHUMOS, AVA" + }, + "evaluation_timestamp": "2024-08-04T12:52:48.336Z" }, { - "evaluation_name": "nuScenes", + "evaluation_name": "DAVIS (Densely Annotated VIdeo Segmentation)", "metric_config": { - "evaluation_description": "nuScenes Standard Accuracy", + "evaluation_description": "DAVIS (Densely Annotated VIdeo Segmentation) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -903,38 +947,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.835355287436681, + "score": 0.49633107694497697, "details": { - "subtask_a": 0.835355287436681, - "subtask_b": 0.835355287436681 + "subtask_a": 0.49633107694497697, + "subtask_b": 0.49633107694497697 } }, "factsheet": { - "purpose": "Research; Development; Deployment", - "principles_tested": "3D object detection, Multi-object tracking, Prediction, Sensor fusion, Scene understanding", + "purpose": "Research; Development", + "principles_tested": "Video object segmentation, Temporal consistency, Object tracking, Dense prediction", "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image); Video; Structured Data", + "input_modality": "Video", "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", "output_source": "Human annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (700), Val (150), Test (150)", + "size": "Small (< 1K samples)", + "splits": "Train/Val (60 sequences), Test-dev (30 sequences)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives multimodal sensor data (cameras, LiDAR, radar) 2. Model predicts 3D bounding boxes and tracking IDs 3. NDS (nuScenes Detection Score) and mAP computed", + "protocol": "1. Model receives video sequence with first frame annotation 2. Model propagates segmentation to subsequent frames 3. J&F metric (region similarity and contour accuracy)", "model_access": "Outputs", "has_heldout": true, - "alignment_validation": "Expert annotators, multi-sensor consistency checks, temporal coherence validation", - "baseline_models": "PointPillars: 45.3 NDS CenterPoint: 65.5 NDS BEVFusion: 72.9 NDS", - "robustness_measures": "Geographic diversity; Weather conditions; Day/night variations; Multiple runs per sample", - "known_limitations": "Limited geographic coverage (Boston, Singapore); Sensor calibration challenges; Annotation latency for fast-moving objects; Class imbalance", - "benchmarks_list": "Waymo Open Dataset, KITTI, Argoverse, Lyft Level 5, A2D2" - } + "heldout_details": "Test-challenge set for competitions, labels withheld", + "alignment_validation": "High-quality manual annotations with temporal consistency verification", + "baseline_models": "OSVOS: 79.8 J&F STM: 84.3 J&F XMem: 86.2 J&F", + "robustness_measures": "Temporal consistency tested; Occlusion robustness; Multiple runs per sample", + "known_limitations": "Small dataset size; Limited object categories; Short video sequences; Primarily objects with clear boundaries", + "benchmarks_list": "YouTube-VOS, FBMS, SegTrack, OVIS" + }, + "evaluation_timestamp": "2024-07-23T22:50:04.114Z" }, { - "evaluation_name": "CIFAR-10 and CIFAR-100", + "evaluation_name": "VQA (Visual Question Answering)", "metric_config": { - "evaluation_description": "CIFAR-10 and CIFAR-100 Standard Accuracy", + "evaluation_description": "VQA (Visual Question Answering) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -942,38 +988,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9542263595592543, + "score": 0.5669783991461111, "details": { - "subtask_a": 0.9542263595592543, - "subtask_b": 0.9542263595592543 + "subtask_a": 0.5669783991461111, + "subtask_b": 0.5669783991461111 } }, "factsheet": { - "purpose": "Research; Development; Selection", - "principles_tested": "Image classification, Transfer learning, Generalization", + "purpose": "Research; Development", + "principles_tested": "Visual question answering, Visual reasoning, Multimodal understanding, Common sense reasoning", "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image)", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (50K), Test (10K)", + "input_modality": "Text + Vision", + "output_modality": "Text", + "input_source": "MS COCO", + "output_source": "Crowdsourced annotations", + "size": "Huge (> 10M samples)", + "splits": "Train, Val, Test", "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives 32x32 RGB image 2. Model predicts class label (10 or 100 classes) 3. Classification accuracy computed", + "judge": "Automatic (Reference-based); Human: Representative sample", + "protocol": "1. Model receives image and question 2. Model generates answer 3. Accuracy computed with consensus matching (multiple human answers)", "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Systematic sampling from larger dataset (80 million tiny images), human verification of labels", - "baseline_models": "CIFAR-10: ResNet-56: 93.03% Wide ResNet-28-10: 96.11% PyramidNet: 96.54% CIFAR-100: ResNet-56: 71.35% Wide ResNet-28-10: 81.15% PyramidNet: 83.78%", - "robustness_measures": "Multiple runs per sample; Seed variation tested; Ablation studies", - "known_limitations": "Low resolution (32x32) limits fine-grained recognition; Some label noise in CIFAR-100; Dataset size enables memorization in large models; Limited diversity in poses and contexts", - "benchmarks_list": "CIFAR-10-C, CIFAR-10.1, CIFAR-100-C, STL-10, Tiny ImageNet" - } + "has_heldout": true, + "heldout_details": "Test-dev and test-std splits, evaluation via server", + "alignment_validation": "Multiple human answers per question (10 answers), consensus-based evaluation", + "baseline_models": "Bottom-Up Top-Down: 70.3% VILBERT: 72.4% OSCAR: 73.8% BLIP: 78.3%", + "robustness_measures": "Multiple human references; Prompt variations tested; Inter-rater reliability", + "known_limitations": "Language bias (can answer many questions without image); Dataset bias toward common objects; Answer distribution imbalance; Ambiguous questions", + "benchmarks_list": "GQA, VQA v2, OK-VQA, TextVQA, VizWiz" + }, + "evaluation_timestamp": "2024-07-13T05:24:53.074Z" }, { - "evaluation_name": "ARC (AI2 Reasoning Challenge)", + "evaluation_name": "CLEVR", "metric_config": { - "evaluation_description": "ARC (AI2 Reasoning Challenge) Standard Accuracy", + "evaluation_description": "CLEVR Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -981,38 +1029,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9930185347075066, + "score": 0.5238089282123, "details": { - "subtask_a": 0.9930185347075066, - "subtask_b": 0.9930185347075066 + "subtask_a": 0.5238089282123, + "subtask_b": 0.5238089282123 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Scientific reasoning, Commonsense reasoning, Knowledge retrieval, Multi-hop reasoning", + "principles_tested": "Compositional reasoning, Spatial reasoning, Counting, Logical reasoning, Visual reasoning", "functional_props": "Core Performance; Robustness", - "input_modality": "Text", + "input_modality": "Text + Vision", "output_modality": "Text", - "input_source": "New dataset (released with eval)", - "output_source": "Existing dataset labels", - "size": "Medium (1K - 100K samples)", - "splits": "Easy (2,376 train, 570 test), Challenge (1,119 train, 1,172 test)", + "input_source": "Synthetic/Generated", + "output_source": "Programmatically generated", + "size": "Huge (> 10M samples)", + "splits": "Train (70K), Val (15K), Test (15K)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives science exam question with multiple choice options 2. Model selects answer 3. Accuracy computed separately for Easy and Challenge sets", + "protocol": "1. Model receives synthetic image and question 2. Model predicts answer from predefined set 3. Accuracy computed, can analyze by question type", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Real exam questions filtered to require reasoning, partitioned by difficulty using retrieval-based baseline", - "baseline_models": "ARC-Challenge: BERT: 59.1% GPT-3: 51.4% GPT-3.5: 85.2% GPT-4: 96.3%", - "robustness_measures": "Two difficulty levels; Multiple choice format variations; Retrieval baseline comparison", - "known_limitations": "Multiple choice format; Some questions solvable without reasoning; Challenge set becoming saturated; Grade-school level may not test advanced reasoning", - "benchmarks_list": "OpenBookQA, CommonsenseQA, QASC, SciQ" - } + "heldout_details": "", + "alignment_validation": "Programmatically generated with known ground truth, exhaustive question type coverage", + "baseline_models": "CNN+LSTM: 52.3% Film: 97.7% NS-VQA: 99.8% MAC: 98.9%", + "robustness_measures": "Ablation studies; Question type analysis; Compositional generalization tested", + "known_limitations": "Synthetic domain (limited real-world applicability); Simple shapes and colors; No natural language variation; Programmatic biases", + "benchmarks_list": "CLEVR-CoGenT, GQA, NLVR2, CLOSURE" + }, + "evaluation_timestamp": "2024-07-24T19:18:36.909Z" }, { - "evaluation_name": "GSM8K (Grade School Math 8K)", + "evaluation_name": "Waymo Open Dataset", "metric_config": { - "evaluation_description": "GSM8K (Grade School Math 8K) Standard Accuracy", + "evaluation_description": "Waymo Open Dataset Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -1020,77 +1070,81 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8227992101886995, + "score": 0.5225249005085325, "details": { - "subtask_a": 0.8227992101886995, - "subtask_b": 0.8227992101886995 + "subtask_a": 0.5225249005085325, + "subtask_b": 0.5225249005085325 } }, "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Mathematical reasoning, Multi-step problem solving, Arithmetic reasoning, Chain-of-thought reasoning", - "functional_props": "Core Performance", - "input_modality": "Text", - "output_modality": "Text", + "purpose": "Research; Development; Deployment", + "principles_tested": "3D object detection, 2D object detection, Tracking, Domain adaptation, Sensor fusion", + "functional_props": "Core Performance; Robustness", + "input_modality": "Vision (Image); Video; Structured Data", + "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Author-provided", - "size": "Medium (1K - 100K samples)", - "splits": "Train (7,473), Test (1,319)", + "output_source": "Human annotations; Programmatically generated", + "size": "Huge (> 10M samples)", + "splits": "Train (798), Val (202), Test (150)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives math word problem 2. Model generates solution with reasoning steps 3. Final numerical answer extracted and compared to ground truth", + "protocol": "1. Model receives multimodal sensor data 2. Model predicts 3D bounding boxes with tracking IDs 3. AP/APH metrics at different IoU thresholds and difficulty levels", "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Human-written problems with verified solutions, consistency checks for answer correctness", - "baseline_models": "GPT-3 (175B): 34.2% GPT-3.5: 57.1% GPT-4: 92.0% GPT-4o: 96.1% Claude 3.5 Sonnet: 96.4%", - "robustness_measures": "Multiple runs per sample; Chain-of-thought prompting tested; Answer extraction robustness", - "known_limitations": "Limited to grade-school level math; Answer extraction can be brittle; Some problems have ambiguous wording; Potential contamination in training data", - "benchmarks_list": "MATH, GSM-Hard, SVAMP, ASDiv, MathQA" - } + "has_heldout": true, + "heldout_details": "Test set labels private, evaluation via online leaderboard", + "alignment_validation": "Multi-stage annotation pipeline with quality checks, temporal consistency validation", + "baseline_models": "PointPillars: 63.8 L2 APH (Vehicle) CenterPoint: 73.9 L2 APH PV-RCNN++: 77.8 L2 APH", + "robustness_measures": "Geographic diversity; Time of day variations; Weather conditions; Multiple difficulty levels", + "known_limitations": "Geographic concentration in specific cities; Sensor-specific challenges; Annotation latency for distant objects; Class imbalance", + "benchmarks_list": "nuScenes, KITTI, Argoverse 2, Once, Lyft Level 5" + }, + "evaluation_timestamp": "2024-08-04T09:58:11.232Z" }, { - "evaluation_name": "MT-Bench", + "evaluation_name": "Fashion-MNIST", "metric_config": { - "evaluation_description": "Multi-turn conversation quality score (1-10)", + "evaluation_description": "Fashion-MNIST Standard Accuracy", "lower_is_better": false, "score_type": "continuous", - "min_score": 1, - "max_score": 10, - "unit": "points" + "min_score": 0, + "max_score": 1, + "unit": "accuracy" }, "score_details": { - "score": 8.759230418819893, + "score": 0.807961023643812, "details": { - "Turn 1": 8.259230418819893, - "Turn 2": 9.259230418819893 + "subtask_a": 0.807961023643812, + "subtask_b": 0.807961023643812 } }, "factsheet": { - "purpose": "Development; Selection", - "principles_tested": "Multi-turn conversation, Instruction following, Reasoning, Writing, Role-playing, Knowledge", - "functional_props": "Core Performance; Core Quality Dimensions", - "input_modality": "Text", - "output_modality": "Text", + "purpose": "Research; Development; Selection", + "principles_tested": "Image classification, Transfer learning, Benchmark comparison", + "functional_props": "Core Performance", + "input_modality": "Vision (Image)", + "output_modality": "Structured Data", "input_source": "New dataset (released with eval)", - "output_source": "Author-provided", - "size": "Small (< 1K samples)", - "splits": "Test (80 questions with 2 turns each)", + "output_source": "Human annotations", + "size": "Medium (1K - 100K samples)", + "splits": "Train (60K), Test (10K)", "design": "Fixed data-driven (static test set)", - "judge": "Model-based: In the wild", - "protocol": "1. Model engages in 2-turn conversation 2. GPT-4 evaluates responses on 10-point scale 3. Scores averaged across turns and categories", + "judge": "Automatic (Reference-based)", + "protocol": "1. Model receives 28x28 grayscale image 2. Model predicts fashion category (10 classes) 3. Classification accuracy computed", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Strong correlation with human preferences validated on Chatbot Arena data, GPT-4 judge agreement measured", - "baseline_models": "Vicuna-13B: 6.39 GPT-3.5-turbo: 7.94 Claude 2: 8.06 GPT-4: 8.99 GPT-4-turbo: 9.32", - "robustness_measures": "Multiple categories; Position bias mitigation; Agreement with human ratings validated; Pairwise comparison", - "known_limitations": "Small dataset (80 questions); GPT-4 judge may have biases; Evaluation cost; Model-based judge limitations; Prompt sensitivity", - "benchmarks_list": "Chatbot Arena, AlpacaEval, Arena-Hard, LiveBench" - } + "heldout_details": "", + "alignment_validation": "Product catalog labels verified by domain experts", + "baseline_models": "Linear Classifier: 83.7% CNN: 93.5% ResNet: 94.9% Vision Transformer: 95.1%", + "robustness_measures": "Multiple runs per sample; Seed variation tested", + "known_limitations": "Low resolution (28x28); Grayscale only; Limited intra-class variation; Some categories overlap visually", + "benchmarks_list": "MNIST, EMNIST, Kuzushiji-MNIST, DeepFashion" + }, + "evaluation_timestamp": "2024-07-23T23:11:33.175Z" }, { - "evaluation_name": "GPQA (Google-Proof Q&A)", + "evaluation_name": "MMLU (Massive Multitask Language Understanding)", "metric_config": { - "evaluation_description": "GPQA (Google-Proof Q&A) Standard Accuracy", + "evaluation_description": "MMLU (Massive Multitask Language Understanding) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -1098,38 +1152,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8998068136536718, + "score": 0.7191302119695253, "details": { - "subtask_a": 0.8998068136536718, - "subtask_b": 0.8998068136536718 + "subtask_a": 0.7191302119695253, + "subtask_b": 0.7191302119695253 } }, "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Expert-level knowledge, Scientific reasoning, Domain expertise, Graduate-level understanding", - "functional_props": "Core Performance; Robustness", + "purpose": "Research; Development; Selection", + "principles_tested": "World knowledge, Reasoning, Domain expertise across 57 subjects", + "functional_props": "Core Performance", "input_modality": "Text", "output_modality": "Text", "input_source": "New dataset (released with eval)", "output_source": "Expert annotations", - "size": "Small (< 1K samples)", - "splits": "Main (198), Extended (246), Diamond (198 highest quality)", + "size": "Medium (1K - 100K samples)", + "splits": "Dev (5 shot examples per task), Test (285 questions per task average)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives graduate-level science question 2. Model selects from 4 multiple choice options 3. Accuracy computed 4. Questions validated to be difficult for non-experts with Google access", + "protocol": "1. Model receives multiple choice question with 4 options 2. Model predicts answer (A/B/C/D) 3. Accuracy computed per subject and overall", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "PhD-level experts write and validate questions, non-expert validators with Google access achieve <35% accuracy", - "baseline_models": "Random: 25% Non-expert w/ Google: 34% Expert: 81% GPT-4: 39% Claude 3 Opus: 59.4% GPT-4o: 53.6%", - "robustness_measures": "Expert validation; Non-expert baseline; Multiple subject areas; Quality tiers (Diamond subset)", - "known_limitations": "Small dataset size; Limited to 3 scientific domains; Multiple choice format; High cost of expert question creation; Cultural bias toward Western scientific education", - "benchmarks_list": "MMLU-Pro, JEE-Advanced, SciBench, LiveBench" - } + "heldout_details": "", + "alignment_validation": "Questions sourced from practice exams and tests, expert review for correctness", + "baseline_models": "GPT-3 (175B): 43.9% GPT-3.5: 70.0% GPT-4: 86.4% Claude 3.5 Sonnet: 88.7% Random baseline: 25%", + "robustness_measures": "Multiple evaluation runs; Few-shot prompting variations; Subject-wise analysis", + "known_limitations": "Multiple choice format may not reflect real-world usage; Some questions have ambiguous answers; Dataset contamination concerns with web-trained models; Cultural and knowledge cutoff biases", + "benchmarks_list": "MMLU-Pro, AGIEval, C-Eval, CMMLU" + }, + "evaluation_timestamp": "2024-08-05T07:28:23.785Z" }, { - "evaluation_name": "COCO (Common Objects in Context)", + "evaluation_name": "HumanEval", "metric_config": { - "evaluation_description": "COCO (Common Objects in Context) Standard Accuracy", + "evaluation_description": "HumanEval Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -1137,38 +1193,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9634483070574376, + "score": 0.607017899246966, "details": { - "subtask_a": 0.9634483070574376, - "subtask_b": 0.9634483070574376 + "subtask_a": 0.607017899246966, + "subtask_b": 0.607017899246966 } }, "factsheet": { "purpose": "Research; Development; Selection", - "principles_tested": "Object detection, Instance segmentation, Keypoint detection, Panoptic segmentation, Image captioning", + "principles_tested": "Code generation, Programming ability, Functional correctness, Code understanding", "functional_props": "Core Performance", - "input_modality": "Vision (Image)", - "output_modality": "Text; Structured Data", + "input_modality": "Text; Code", + "output_modality": "Code", "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", - "size": "Large (100K - 1M samples)", - "splits": "Train, Validation, Test", + "output_source": "Author-provided", + "size": "Small (< 1K samples)", + "splits": "Test (164 problems)", "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives image as input 2. Model outputs bounding boxes, segmentation masks, or captions 3. Metrics computed: mAP for detection, IoU for segmentation, BLEU/CIDEr for captioning", + "judge": "Automatic (Execution-based)", + "protocol": "1. Model receives function signature and docstring 2. Model generates function implementation 3. Generated code executed against unit tests 4. pass@k metric computed (% passing all tests in k samples)", "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Multi-annotator consensus for instance annotations, quality control through redundant labeling", - "baseline_models": "Faster R-CNN: 42.0 mAP Mask R-CNN: 37.1 mask mAP YOLOv8: 53.9 mAP Human performance (detection): ~70 mAP", - "robustness_measures": "Inter-rater reliability; Multiple runs per sample; Confidence intervals", - "known_limitations": "Small object detection remains challenging; Occlusion handling difficulties; Dataset bias toward certain object contexts; Annotation inconsistencies in crowded scenes", - "benchmarks_list": "LVIS, Objects365, Open Images, Visual Genome" - } + "has_heldout": false, + "heldout_details": "", + "alignment_validation": "Hand-written problems with comprehensive unit tests, manual verification of test correctness", + "baseline_models": "Codex (12B): 28.8% pass@1 GPT-3.5-turbo: 48.1% pass@1 GPT-4: 67.0% pass@1 Claude 3.5 Sonnet: 92.0% pass@1", + "robustness_measures": "Multiple samples per problem (pass@k); Temperature sensitivity tested; Execution-based verification", + "known_limitations": "Limited to Python; Small dataset size (164 problems); Relatively simple problems; May be contaminated in training data; No testing of code efficiency or style", + "benchmarks_list": "MBPP, APPS, CodeContests, HumanEval+, MultiPL-E" + }, + "evaluation_timestamp": "2024-07-27T21:25:17.874Z" }, { - "evaluation_name": "IFEval (Instruction-Following Eval)", + "evaluation_name": "HellaSwag", "metric_config": { - "evaluation_description": "IFEval (Instruction-Following Eval) Standard Accuracy", + "evaluation_description": "HellaSwag Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -1176,38 +1234,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9007200045619222, + "score": 0.7199321337017288, "details": { - "subtask_a": 0.9007200045619222, - "subtask_b": 0.9007200045619222 + "subtask_a": 0.7199321337017288, + "subtask_b": 0.7199321337017288 } }, "factsheet": { - "purpose": "Research; Development; Selection", - "principles_tested": "Instruction following, Constraint satisfaction, Format compliance, Precise control", + "purpose": "Research; Development", + "principles_tested": "Commonsense reasoning, Physical understanding, Situation modeling, Plausibility judgment", "functional_props": "Core Performance; Robustness", "input_modality": "Text", "output_modality": "Text", "input_source": "New dataset (released with eval)", - "output_source": "Programmatically generated", - "size": "Small (< 1K samples)", - "splits": "Test (541 prompts with ~25 verifiable instructions)", + "output_source": "Crowdsourced annotations", + "size": "Medium (1K - 100K samples)", + "splits": "Train (39,905), Val (10,042), Test (10,003)", "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-free)", - "protocol": "1. Model receives prompt with verifiable instructions (e.g., 'respond in exactly 3 paragraphs', 'include word X at least 5 times') 2. Model generates response 3. Programmatic checks verify instruction compliance 4. Strict and loose accuracy metrics computed", + "judge": "Automatic (Reference-based)", + "protocol": "1. Model receives scenario context 2. Model selects most plausible continuation from 4 options 3. Accuracy computed", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Verifiable instructions with programmatic checking, no ambiguity in correctness", - "baseline_models": "GPT-3.5: 57.4% strict GPT-4: 76.9% strict Claude 2: 66.7% strict Gemini Ultra: 79.4% strict", - "robustness_measures": "Strict and loose metrics; Multiple instruction types; Programmatic verification; No human evaluation needed", - "known_limitations": "Limited to verifiable instructions only; May not reflect realistic usage; Some instructions may be conflicting; Excludes semantic quality assessment", - "benchmarks_list": "MT-Bench, AlpacaEval, InstructGPT evals, FollowBench" - } + "heldout_details": "", + "alignment_validation": "Adversarial filtering using BERT to ensure difficulty, human validation of plausibility", + "baseline_models": "BERT-Large: 47.3% GPT-2: 50.9% GPT-3: 78.9% GPT-4: 95.3% Human performance: 95.6%", + "robustness_measures": "Adversarial filtering; Multiple runs per sample; Human baseline comparison", + "known_limitations": "Dataset may be easier than originally intended for modern LLMs; Multiple choice format; Adversarial examples may have artifacts; Potential data contamination", + "benchmarks_list": "PIQA, WinoGrande, CommonsenseQA, ARC" + }, + "evaluation_timestamp": "2024-07-31T06:15:51.311Z" }, { - "evaluation_name": "ScanNet", + "evaluation_name": "GSM8K (Grade School Math 8K)", "metric_config": { - "evaluation_description": "ScanNet Standard Accuracy", + "evaluation_description": "GSM8K (Grade School Math 8K) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -1215,38 +1275,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.829656301826159, + "score": 0.5299845687235446, "details": { - "subtask_a": 0.829656301826159, - "subtask_b": 0.829656301826159 + "subtask_a": 0.5299845687235446, + "subtask_b": 0.5299845687235446 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "3D scene understanding, Semantic segmentation, Instance segmentation, 3D reconstruction", + "principles_tested": "Mathematical reasoning, Multi-step problem solving, Arithmetic reasoning, Chain-of-thought reasoning", "functional_props": "Core Performance", - "input_modality": "Vision (Image); Video; Structured Data", - "output_modality": "Structured Data", + "input_modality": "Text", + "output_modality": "Text", "input_source": "New dataset (released with eval)", - "output_source": "Human annotations; Programmatically generated", + "output_source": "Author-provided", "size": "Medium (1K - 100K samples)", - "splits": "Train (1201), Val (312), Test (100)", + "splits": "Train (7,473), Test (1,319)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives 3D scene (RGB-D scans) 2. Model predicts semantic labels or instance masks 3. mIoU for semantic segmentation, mAP for instance segmentation", + "protocol": "1. Model receives math word problem 2. Model generates solution with reasoning steps 3. Final numerical answer extracted and compared to ground truth", "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Manual verification of 3D reconstructions, multi-annotator consistency for semantic labels", - "baseline_models": "PointNet++: 53.5 mIoU SparseConvNet: 72.5 mIoU MinkowskiNet: 73.6 mIoU", - "robustness_measures": "Multiple runs per sample; Ablation studies", - "known_limitations": "Limited to indoor scenes; Reconstruction artifacts; Scanning noise and occlusions; Limited scene diversity (mostly offices and apartments)", - "benchmarks_list": "Matterport3D, S3DIS, 2D-3D-S, ARKitScenes" - } + "has_heldout": false, + "heldout_details": "", + "alignment_validation": "Human-written problems with verified solutions, consistency checks for answer correctness", + "baseline_models": "GPT-3 (175B): 34.2% GPT-3.5: 57.1% GPT-4: 92.0% GPT-4o: 96.1% Claude 3.5 Sonnet: 96.4%", + "robustness_measures": "Multiple runs per sample; Chain-of-thought prompting tested; Answer extraction robustness", + "known_limitations": "Limited to grade-school level math; Answer extraction can be brittle; Some problems have ambiguous wording; Potential contamination in training data", + "benchmarks_list": "MATH, GSM-Hard, SVAMP, ASDiv, MathQA" + }, + "evaluation_timestamp": "2024-07-30T10:18:34.641Z" }, { - "evaluation_name": "ImageNet", + "evaluation_name": "BIG-bench (Beyond the Imitation Game)", "metric_config": { - "evaluation_description": "ImageNet Standard Accuracy", + "evaluation_description": "BIG-bench (Beyond the Imitation Game) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -1254,72 +1316,35 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8113182704323655, + "score": 0.7123971781869675, "details": { - "subtask_a": 0.8113182704323655, - "subtask_b": 0.8113182704323655 + "subtask_a": 0.7123971781869675, + "subtask_b": 0.7123971781869675 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Object recognition, Visual categorization, Multi-class classification", - "functional_props": "Core Performance", - "input_modality": "Vision (Image)", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", - "size": "Huge (> 10M samples)", - "splits": "Train, Validation, Test", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives an image as input 2. Model predicts class label from 1000 classes 3. Top-1 and Top-5 accuracy computed against ground truth", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Expert curation of hierarchical categories, manual verification of labels, consistency checks across similar categories", - "baseline_models": "AlexNet: 63.3% top-1 VGG: 71.5% top-1 ResNet-50: 76.1% top-1 ResNet-152: 78.3% top-1 EfficientNet-B7: 84.3% top-1 Human performance: ~95% top-5", - "robustness_measures": "Multiple runs per sample; Significance testing; Inter-rater reliability", - "known_limitations": "Class imbalance in some categories; Label noise in training set; Some ambiguous images with multiple valid labels; Bias towards certain object viewpoints and contexts", - "benchmarks_list": "ImageNet-V2, ImageNet-C, ImageNet-R, ImageNet-A, ImageNet-Sketch" - } - }, - { - "evaluation_name": "HumanEval", - "metric_config": { - "evaluation_description": "Pass@1 Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "pass@1" - }, - "score_details": { - "score": 0.8221544556154244, - "details": { - "subtask_a": 0.8221544556154244, - "subtask_b": 0.8221544556154244 - } - }, - "factsheet": { - "purpose": "Research; Development; Selection", - "principles_tested": "Code generation, Programming ability, Functional correctness, Code understanding", - "functional_props": "Core Performance", - "input_modality": "Text; Code", - "output_modality": "Code", + "principles_tested": "Diverse capabilities across 204 tasks including reasoning, knowledge, language understanding, bias detection", + "functional_props": "Core Performance; Fairness; Robustness", + "input_modality": "Text", + "output_modality": "Text", "input_source": "New dataset (released with eval)", - "output_source": "Author-provided", - "size": "Small (< 1K samples)", - "splits": "Test (164 problems)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Execution-based)", - "protocol": "1. Model receives function signature and docstring 2. Model generates function implementation 3. Generated code executed against unit tests 4. pass@k metric computed (% passing all tests in k samples)", + "output_source": "Multiple/Mixed sources", + "size": "Large (100K - 1M samples)", + "splits": "Varies by task", + "design": "Composite", + "judge": "Automatic (Reference-based); Model-based: In the wild", + "protocol": "1. Model evaluated on 204 diverse tasks 2. Each task has its own evaluation protocol 3. Performance aggregated across tasks", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Hand-written problems with comprehensive unit tests, manual verification of test correctness", - "baseline_models": "Codex (12B): 28.8% pass@1 GPT-3.5-turbo: 48.1% pass@1 GPT-4: 67.0% pass@1 Claude 3.5 Sonnet: 92.0% pass@1", - "robustness_measures": "Multiple samples per problem (pass@k); Temperature sensitivity tested; Execution-based verification", - "known_limitations": "Limited to Python; Small dataset size (164 problems); Relatively simple problems; May be contaminated in training data; No testing of code efficiency or style", - "benchmarks_list": "MBPP, APPS, CodeContests, HumanEval+, MultiPL-E" - } + "heldout_details": "", + "alignment_validation": "Crowdsourced task creation with quality review, diverse authorship for broad coverage", + "baseline_models": "Average human rater: 89.0% Few-shot PaLM (540B): 65.7% GPT-4: ~83% (estimated on BIG-Bench Hard)", + "robustness_measures": "Multiple tasks provide robustness; Human baseline comparison; Cross-task analysis", + "known_limitations": "Task quality varies; Some tasks too easy or too hard; Computational cost of running all 204 tasks; Aggregation methodology debatable", + "benchmarks_list": "BIG-Bench Hard, MMLU, HELM, SuperGLUE" + }, + "evaluation_timestamp": "2024-07-29T04:52:53.539Z" }, { "evaluation_name": "MATH", @@ -1332,10 +1357,10 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8253062771877044, + "score": 0.4556635278553191, "details": { - "subtask_a": 0.8253062771877044, - "subtask_b": 0.8253062771877044 + "subtask_a": 0.4556635278553191, + "subtask_b": 0.4556635278553191 } }, "factsheet": { @@ -1353,17 +1378,19 @@ "protocol": "1. Model receives competition math problem 2. Model generates solution with steps 3. Final answer extracted and checked against ground truth 4. Problems span 7 subjects with 5 difficulty levels", "model_access": "Outputs", "has_heldout": false, + "heldout_details": "", "alignment_validation": "Problems from real math competitions with verified solutions, difficulty levels validated", "baseline_models": "GPT-3: 6.9% GPT-4: 42.5% Minerva (540B): 33.6% GPT-4 Turbo: 52.9% Claude 3.5 Sonnet: 71.1%", "robustness_measures": "Multiple difficulty levels; Subject-wise analysis; Chain-of-thought evaluation", "known_limitations": "Answer extraction challenges; LaTeX formatting issues; Symbolic vs numeric answers; High difficulty may not reflect practical math usage", "benchmarks_list": "GSM8K, MathQA, SVAMP, ASDiv, Hendrycks MATH" - } + }, + "evaluation_timestamp": "2024-07-31T17:34:31.032Z" }, { - "evaluation_name": "IntPhys 2", + "evaluation_name": "ARC (AI2 Reasoning Challenge)", "metric_config": { - "evaluation_description": "IntPhys 2 Standard Accuracy", + "evaluation_description": "ARC (AI2 Reasoning Challenge) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -1371,662 +1398,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8514689801139111, + "score": 0.6381512167876835, "details": { - "subtask_a": 0.8514689801139111, - "subtask_b": 0.8514689801139111 + "subtask_a": 0.6381512167876835, + "subtask_b": 0.6381512167876835 } }, "factsheet": { - "purpose": "Research", - "principles_tested": "Intuitive Physics", - "functional_props": "Core Performance", - "input_modality": "Video", - "output_modality": "Scores/Embeddings", + "purpose": "Research; Development", + "principles_tested": "Scientific reasoning, Commonsense reasoning, Knowledge retrieval, Multi-hop reasoning", + "functional_props": "Core Performance; Robustness", + "input_modality": "Text", + "output_modality": "Text", "input_source": "New dataset (released with eval)", - "output_source": "Simulation-based", + "output_source": "Existing dataset labels", "size": "Medium (1K - 100K samples)", - "splits": "Debug Set: 60 videos for Model calibration Main Set: 1,012 videos as Main evaluation set Held-Out Set: 344 videos as Test set", + "splits": "Easy (2,376 train, 570 test), Challenge (1,119 train, 1,172 test)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "Feed a video to the model Ask the model using specific prompts wether the video is physically plausible or not Check if the model's answer match the ground truth label", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "We ran human baselines to ensure that this task is easy for humans.", - "baseline_models": "- Human baseline: 96% - Best model baseline: V-JEPA2 56%", - "robustness_measures": "Prompt variations tested; Multiple runs per sample; Temperature sensitivity tested", - "known_limitations": "Model can be very sensitive to the way they are prompted. Also the compression artefact of the video can also impact the results. Those are mostly limitations on the model sides, since human are not sensitive to those.", - "benchmarks_list": "IntPhys: A Framework and Benchmark for Visual Intuitive Physics Reasoning, Riochet et al. 2020" - } - }, - { - "evaluation_name": "MT-Bench", - "metric_config": { - "evaluation_description": "Multi-turn conversation quality score (1-10)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 1, - "max_score": 10, - "unit": "points" - }, - "score_details": { - "score": 8.586649753403101, - "details": { - "Turn 1": 8.086649753403101, - "Turn 2": 9.086649753403101 - } - }, - "factsheet": { - "purpose": "Development; Selection", - "principles_tested": "Multi-turn conversation, Instruction following, Reasoning, Writing, Role-playing, Knowledge", - "functional_props": "Core Performance; Core Quality Dimensions", - "input_modality": "Text", - "output_modality": "Text", - "input_source": "New dataset (released with eval)", - "output_source": "Author-provided", - "size": "Small (< 1K samples)", - "splits": "Test (80 questions with 2 turns each)", - "design": "Fixed data-driven (static test set)", - "judge": "Model-based: In the wild", - "protocol": "1. Model engages in 2-turn conversation 2. GPT-4 evaluates responses on 10-point scale 3. Scores averaged across turns and categories", - "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Strong correlation with human preferences validated on Chatbot Arena data, GPT-4 judge agreement measured", - "baseline_models": "Vicuna-13B: 6.39 GPT-3.5-turbo: 7.94 Claude 2: 8.06 GPT-4: 8.99 GPT-4-turbo: 9.32", - "robustness_measures": "Multiple categories; Position bias mitigation; Agreement with human ratings validated; Pairwise comparison", - "known_limitations": "Small dataset (80 questions); GPT-4 judge may have biases; Evaluation cost; Model-based judge limitations; Prompt sensitivity", - "benchmarks_list": "Chatbot Arena, AlpacaEval, Arena-Hard, LiveBench" - } - }, - { - "evaluation_name": "IFEval (Instruction-Following Eval)", - "metric_config": { - "evaluation_description": "IFEval (Instruction-Following Eval) Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.8101965007854924, - "details": { - "subtask_a": 0.8101965007854924, - "subtask_b": 0.8101965007854924 - } - }, - "factsheet": { - "purpose": "Research; Development; Selection", - "principles_tested": "Instruction following, Constraint satisfaction, Format compliance, Precise control", - "functional_props": "Core Performance; Robustness", - "input_modality": "Text", - "output_modality": "Text", - "input_source": "New dataset (released with eval)", - "output_source": "Programmatically generated", - "size": "Small (< 1K samples)", - "splits": "Test (541 prompts with ~25 verifiable instructions)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-free)", - "protocol": "1. Model receives prompt with verifiable instructions (e.g., 'respond in exactly 3 paragraphs', 'include word X at least 5 times') 2. Model generates response 3. Programmatic checks verify instruction compliance 4. Strict and loose accuracy metrics computed", - "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Verifiable instructions with programmatic checking, no ambiguity in correctness", - "baseline_models": "GPT-3.5: 57.4% strict GPT-4: 76.9% strict Claude 2: 66.7% strict Gemini Ultra: 79.4% strict", - "robustness_measures": "Strict and loose metrics; Multiple instruction types; Programmatic verification; No human evaluation needed", - "known_limitations": "Limited to verifiable instructions only; May not reflect realistic usage; Some instructions may be conflicting; Excludes semantic quality assessment", - "benchmarks_list": "MT-Bench, AlpacaEval, InstructGPT evals, FollowBench" - } - }, - { - "evaluation_name": "CIFAR-10 and CIFAR-100", - "metric_config": { - "evaluation_description": "CIFAR-10 and CIFAR-100 Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.8142172012513299, - "details": { - "subtask_a": 0.8142172012513299, - "subtask_b": 0.8142172012513299 - } - }, - "factsheet": { - "purpose": "Research; Development; Selection", - "principles_tested": "Image classification, Transfer learning, Generalization", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image)", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (50K), Test (10K)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives 32x32 RGB image 2. Model predicts class label (10 or 100 classes) 3. Classification accuracy computed", - "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Systematic sampling from larger dataset (80 million tiny images), human verification of labels", - "baseline_models": "CIFAR-10: ResNet-56: 93.03% Wide ResNet-28-10: 96.11% PyramidNet: 96.54% CIFAR-100: ResNet-56: 71.35% Wide ResNet-28-10: 81.15% PyramidNet: 83.78%", - "robustness_measures": "Multiple runs per sample; Seed variation tested; Ablation studies", - "known_limitations": "Low resolution (32x32) limits fine-grained recognition; Some label noise in CIFAR-100; Dataset size enables memorization in large models; Limited diversity in poses and contexts", - "benchmarks_list": "CIFAR-10-C, CIFAR-10.1, CIFAR-100-C, STL-10, Tiny ImageNet" - } - }, - { - "evaluation_name": "LVIS (Large Vocabulary Instance Segmentation)", - "metric_config": { - "evaluation_description": "LVIS (Large Vocabulary Instance Segmentation) Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.9998959750760711, - "details": { - "subtask_a": 0.9998959750760711, - "subtask_b": 0.9998959750760711 - } - }, - "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Instance segmentation, Long-tail recognition, Object detection, Fine-grained categorization", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image)", - "output_modality": "Structured Data", - "input_source": "MS COCO", - "output_source": "Expert annotations", - "size": "Large (100K - 1M samples)", - "splits": "Train, Val, Test", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives image 2. Model predicts instance masks and categories (1203 classes) 3. AP computed separately for rare, common, and frequent categories", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Expert annotators with WordNet taxonomy, quality control for long-tail categories", - "baseline_models": "Mask R-CNN: 21.2 AP (v1.0) Cascade R-CNN: 26.2 AP Swin Transformer: 50.9 AP", - "robustness_measures": "Multiple runs per sample; Ablation studies; Category frequency stratification", - "known_limitations": "Rare categories have very few examples; Annotation cost for 1203 categories; Some category definitions overlap; Challenging for zero-shot generalization", - "benchmarks_list": "COCO, Objects365, OpenImages, iNaturalist" - } - }, - { - "evaluation_name": "Open Images", - "metric_config": { - "evaluation_description": "Open Images Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.8334939903755666, - "details": { - "subtask_a": 0.8334939903755666, - "subtask_b": 0.8334939903755666 - } - }, - "factsheet": { - "purpose": "Research; Development; Selection", - "principles_tested": "Object detection, Image classification, Visual relationship detection, Instance segmentation", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image)", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations; Crowdsourced annotations", - "size": "Very Huge (> 100M samples)", - "splits": "Train, Val, Test", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives image 2. Model performs classification, detection, or segmentation 3. mAP computed for detection, top-k accuracy for classification", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Multi-stage crowdsourced annotation with verification, automated quality filters", - "baseline_models": "Faster R-CNN: 54.3 mAP (detection) YOLOv4: 55.8 mAP EfficientDet: 56.1 mAP", - "robustness_measures": "Multiple runs per sample; Confidence intervals; Large-scale diversity", - "known_limitations": "Long-tail class distribution; Annotation inconsistencies at scale; Some classes poorly defined; Label noise in crowdsourced annotations", - "benchmarks_list": "COCO, LVIS, Objects365, ImageNet" - } - }, - { - "evaluation_name": "HellaSwag", - "metric_config": { - "evaluation_description": "HellaSwag Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.9142839736575727, - "details": { - "subtask_a": 0.9142839736575727, - "subtask_b": 0.9142839736575727 - } - }, - "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Commonsense reasoning, Physical understanding, Situation modeling, Plausibility judgment", - "functional_props": "Core Performance; Robustness", - "input_modality": "Text", - "output_modality": "Text", - "input_source": "New dataset (released with eval)", - "output_source": "Crowdsourced annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (39,905), Val (10,042), Test (10,003)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives scenario context 2. Model selects most plausible continuation from 4 options 3. Accuracy computed", - "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Adversarial filtering using BERT to ensure difficulty, human validation of plausibility", - "baseline_models": "BERT-Large: 47.3% GPT-2: 50.9% GPT-3: 78.9% GPT-4: 95.3% Human performance: 95.6%", - "robustness_measures": "Adversarial filtering; Multiple runs per sample; Human baseline comparison", - "known_limitations": "Dataset may be easier than originally intended for modern LLMs; Multiple choice format; Adversarial examples may have artifacts; Potential data contamination", - "benchmarks_list": "PIQA, WinoGrande, CommonsenseQA, ARC" - } - }, - { - "evaluation_name": "Cityscapes", - "metric_config": { - "evaluation_description": "Cityscapes Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.8139990726008225, - "details": { - "subtask_a": 0.8139990726008225, - "subtask_b": 0.8139990726008225 - } - }, - "factsheet": { - "purpose": "Research; Development; Deployment", - "principles_tested": "Semantic segmentation, Instance segmentation, Scene understanding, Autonomous driving perception", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image); Video", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Expert annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (2975), Val (500), Test (1525)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives street scene image (2048×1024) 2. Model predicts per-pixel semantic class (19 or 30 classes) 3. IoU and accuracy metrics computed", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Expert annotators with domain knowledge, multi-pass quality control, consistency verification across video sequences", - "baseline_models": "FCN-8s: 65.3 mIoU DeepLab v3+: 82.1 mIoU HRNetV2: 83.0 mIoU SegFormer: 84.0 mIoU", - "robustness_measures": "Multiple runs per sample; Ablation studies; Significance testing", - "known_limitations": "Limited to European cities; Weather bias (mostly good conditions); Class imbalance for rare objects; Fine annotation boundaries challenging", - "benchmarks_list": "ADE20K, KITTI, Mapillary Vistas, BDD100K, nuScenes" - } - }, - { - "evaluation_name": "nuScenes", - "metric_config": { - "evaluation_description": "nuScenes Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.9914143672799767, - "details": { - "subtask_a": 0.9914143672799767, - "subtask_b": 0.9914143672799767 - } - }, - "factsheet": { - "purpose": "Research; Development; Deployment", - "principles_tested": "3D object detection, Multi-object tracking, Prediction, Sensor fusion, Scene understanding", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image); Video; Structured Data", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (700), Val (150), Test (150)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives multimodal sensor data (cameras, LiDAR, radar) 2. Model predicts 3D bounding boxes and tracking IDs 3. NDS (nuScenes Detection Score) and mAP computed", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Expert annotators, multi-sensor consistency checks, temporal coherence validation", - "baseline_models": "PointPillars: 45.3 NDS CenterPoint: 65.5 NDS BEVFusion: 72.9 NDS", - "robustness_measures": "Geographic diversity; Weather conditions; Day/night variations; Multiple runs per sample", - "known_limitations": "Limited geographic coverage (Boston, Singapore); Sensor calibration challenges; Annotation latency for fast-moving objects; Class imbalance", - "benchmarks_list": "Waymo Open Dataset, KITTI, Argoverse, Lyft Level 5, A2D2" - } - }, - { - "evaluation_name": "CLEVR", - "metric_config": { - "evaluation_description": "CLEVR Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.8472282372976484, - "details": { - "subtask_a": 0.8472282372976484, - "subtask_b": 0.8472282372976484 - } - }, - "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Compositional reasoning, Spatial reasoning, Counting, Logical reasoning, Visual reasoning", - "functional_props": "Core Performance; Robustness", - "input_modality": "Text + Vision", - "output_modality": "Text", - "input_source": "Synthetic/Generated", - "output_source": "Programmatically generated", - "size": "Huge (> 10M samples)", - "splits": "Train (70K), Val (15K), Test (15K)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives synthetic image and question 2. Model predicts answer from predefined set 3. Accuracy computed, can analyze by question type", - "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Programmatically generated with known ground truth, exhaustive question type coverage", - "baseline_models": "CNN+LSTM: 52.3% Film: 97.7% NS-VQA: 99.8% MAC: 98.9%", - "robustness_measures": "Ablation studies; Question type analysis; Compositional generalization tested", - "known_limitations": "Synthetic domain (limited real-world applicability); Simple shapes and colors; No natural language variation; Programmatic biases", - "benchmarks_list": "CLEVR-CoGenT, GQA, NLVR2, CLOSURE" - } - }, - { - "evaluation_name": "Waymo Open Dataset", - "metric_config": { - "evaluation_description": "Waymo Open Dataset Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.9844730411952066, - "details": { - "subtask_a": 0.9844730411952066, - "subtask_b": 0.9844730411952066 - } - }, - "factsheet": { - "purpose": "Research; Development; Deployment", - "principles_tested": "3D object detection, 2D object detection, Tracking, Domain adaptation, Sensor fusion", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image); Video; Structured Data", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations; Programmatically generated", - "size": "Huge (> 10M samples)", - "splits": "Train (798), Val (202), Test (150)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives multimodal sensor data 2. Model predicts 3D bounding boxes with tracking IDs 3. AP/APH metrics at different IoU thresholds and difficulty levels", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Multi-stage annotation pipeline with quality checks, temporal consistency validation", - "baseline_models": "PointPillars: 63.8 L2 APH (Vehicle) CenterPoint: 73.9 L2 APH PV-RCNN++: 77.8 L2 APH", - "robustness_measures": "Geographic diversity; Time of day variations; Weather conditions; Multiple difficulty levels", - "known_limitations": "Geographic concentration in specific cities; Sensor-specific challenges; Annotation latency for distant objects; Class imbalance", - "benchmarks_list": "nuScenes, KITTI, Argoverse 2, Once, Lyft Level 5" - } - }, - { - "evaluation_name": "BIG-bench (Beyond the Imitation Game)", - "metric_config": { - "evaluation_description": "BIG-bench (Beyond the Imitation Game) Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.9998551412963931, - "details": { - "subtask_a": 0.9998551412963931, - "subtask_b": 0.9998551412963931 - } - }, - "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Diverse capabilities across 204 tasks including reasoning, knowledge, language understanding, bias detection", - "functional_props": "Core Performance; Fairness; Robustness", - "input_modality": "Text", - "output_modality": "Text", - "input_source": "New dataset (released with eval)", - "output_source": "Multiple/Mixed sources", - "size": "Large (100K - 1M samples)", - "splits": "Varies by task", - "design": "Composite", - "judge": "Automatic (Reference-based); Model-based: In the wild", - "protocol": "1. Model evaluated on 204 diverse tasks 2. Each task has its own evaluation protocol 3. Performance aggregated across tasks", - "model_access": "Outputs", - "has_heldout": false, - "alignment_validation": "Crowdsourced task creation with quality review, diverse authorship for broad coverage", - "baseline_models": "Average human rater: 89.0% Few-shot PaLM (540B): 65.7% GPT-4: ~83% (estimated on BIG-Bench Hard)", - "robustness_measures": "Multiple tasks provide robustness; Human baseline comparison; Cross-task analysis", - "known_limitations": "Task quality varies; Some tasks too easy or too hard; Computational cost of running all 204 tasks; Aggregation methodology debatable", - "benchmarks_list": "BIG-Bench Hard, MMLU, HELM, SuperGLUE" - } - }, - { - "evaluation_name": "ARC (AI2 Reasoning Challenge)", - "metric_config": { - "evaluation_description": "ARC (AI2 Reasoning Challenge) Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.8411000288481001, - "details": { - "subtask_a": 0.8411000288481001, - "subtask_b": 0.8411000288481001 - } - }, - "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Scientific reasoning, Commonsense reasoning, Knowledge retrieval, Multi-hop reasoning", - "functional_props": "Core Performance; Robustness", - "input_modality": "Text", - "output_modality": "Text", - "input_source": "New dataset (released with eval)", - "output_source": "Existing dataset labels", - "size": "Medium (1K - 100K samples)", - "splits": "Easy (2,376 train, 570 test), Challenge (1,119 train, 1,172 test)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives science exam question with multiple choice options 2. Model selects answer 3. Accuracy computed separately for Easy and Challenge sets", + "protocol": "1. Model receives science exam question with multiple choice options 2. Model selects answer 3. Accuracy computed separately for Easy and Challenge sets", "model_access": "Outputs", "has_heldout": false, + "heldout_details": "", "alignment_validation": "Real exam questions filtered to require reasoning, partitioned by difficulty using retrieval-based baseline", "baseline_models": "ARC-Challenge: BERT: 59.1% GPT-3: 51.4% GPT-3.5: 85.2% GPT-4: 96.3%", - "robustness_measures": "Two difficulty levels; Multiple choice format variations; Retrieval baseline comparison", - "known_limitations": "Multiple choice format; Some questions solvable without reasoning; Challenge set becoming saturated; Grade-school level may not test advanced reasoning", - "benchmarks_list": "OpenBookQA, CommonsenseQA, QASC, SciQ" - } - }, - { - "evaluation_name": "DAVIS (Densely Annotated VIdeo Segmentation)", - "metric_config": { - "evaluation_description": "DAVIS (Densely Annotated VIdeo Segmentation) Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.9350033856486981, - "details": { - "subtask_a": 0.9350033856486981, - "subtask_b": 0.9350033856486981 - } - }, - "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Video object segmentation, Temporal consistency, Object tracking, Dense prediction", - "functional_props": "Core Performance; Robustness", - "input_modality": "Video", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", - "size": "Small (< 1K samples)", - "splits": "Train/Val (60 sequences), Test-dev (30 sequences)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives video sequence with first frame annotation 2. Model propagates segmentation to subsequent frames 3. J&F metric (region similarity and contour accuracy)", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "High-quality manual annotations with temporal consistency verification", - "baseline_models": "OSVOS: 79.8 J&F STM: 84.3 J&F XMem: 86.2 J&F", - "robustness_measures": "Temporal consistency tested; Occlusion robustness; Multiple runs per sample", - "known_limitations": "Small dataset size; Limited object categories; Short video sequences; Primarily objects with clear boundaries", - "benchmarks_list": "YouTube-VOS, FBMS, SegTrack, OVIS" - } - }, - { - "evaluation_name": "KITTI", - "metric_config": { - "evaluation_description": "KITTI Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.8225348119210047, - "details": { - "subtask_a": 0.8225348119210047, - "subtask_b": 0.8225348119210047 - } - }, - "factsheet": { - "purpose": "Research; Development; Deployment", - "principles_tested": "3D object detection, Stereo vision, Optical flow, Visual odometry, Tracking", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image); Video", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations; Programmatically generated", - "size": "Medium (1K - 100K samples)", - "splits": "Train, Val, Test", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives stereo image pair or point cloud 2. Model predicts 3D bounding boxes, orientation, class 3. 3D AP computed at different difficulty levels (easy/moderate/hard)", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "LiDAR ground truth for 3D positions, manual verification of annotations, multi-sensor fusion for accuracy", - "baseline_models": "PointPillars: 79.05 AP (Car, Moderate) PV-RCNN: 83.90 AP (Car, Moderate) CenterPoint: 85.15 AP (Car, Moderate)", - "robustness_measures": "Multiple difficulty levels; Ablation studies; Significance testing", - "known_limitations": "Limited to specific geographic region; Weather bias (mostly clear); Limited nighttime data; Class imbalance (cars dominate)", - "benchmarks_list": "nuScenes, Waymo Open Dataset, Argoverse, A2D2, Lyft Level 5" - } - }, - { - "evaluation_name": "Mapillary Vistas", - "metric_config": { - "evaluation_description": "Mapillary Vistas Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.9225474088472332, - "details": { - "subtask_a": 0.9225474088472332, - "subtask_b": 0.9225474088472332 - } - }, - "factsheet": { - "purpose": "Research; Development; Deployment", - "principles_tested": "Semantic segmentation, Panoptic segmentation, Scene understanding, Robust perception", - "functional_props": "Core Performance; Robustness", - "input_modality": "Vision (Image)", - "output_modality": "Structured Data", - "input_source": "User-generated content", - "output_source": "Expert annotations", - "size": "Medium (1K - 100K samples)", - "splits": "Train (18K), Val (2K), Test (5K)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives street-level image 2. Model predicts per-pixel semantic labels (66 classes) 3. mIoU computed across all classes", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Expert annotators, multi-stage quality control, geographic diversity validation", - "baseline_models": "PSPNet: 42.7 mIoU DeepLab v3+: 45.8 mIoU HRNetV2: 50.3 mIoU", - "robustness_measures": "Geographic diversity tested; Weather variations included; Multiple runs per sample", - "known_limitations": "Varying image quality from crowdsourced data; Camera parameter diversity; Some regions overrepresented; Annotation inconsistencies across diverse scenes", - "benchmarks_list": "Cityscapes, BDD100K, IDD, WildDash" - } - }, - { - "evaluation_name": "VQA (Visual Question Answering)", - "metric_config": { - "evaluation_description": "VQA (Visual Question Answering) Standard Accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1, - "unit": "accuracy" - }, - "score_details": { - "score": 0.9705478098864005, - "details": { - "subtask_a": 0.9705478098864005, - "subtask_b": 0.9705478098864005 - } - }, - "factsheet": { - "purpose": "Research; Development", - "principles_tested": "Visual question answering, Visual reasoning, Multimodal understanding, Common sense reasoning", - "functional_props": "Core Performance; Robustness", - "input_modality": "Text + Vision", - "output_modality": "Text", - "input_source": "MS COCO", - "output_source": "Crowdsourced annotations", - "size": "Huge (> 10M samples)", - "splits": "Train, Val, Test", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based); Human: Representative sample", - "protocol": "1. Model receives image and question 2. Model generates answer 3. Accuracy computed with consensus matching (multiple human answers)", - "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "Multiple human answers per question (10 answers), consensus-based evaluation", - "baseline_models": "Bottom-Up Top-Down: 70.3% VILBERT: 72.4% OSCAR: 73.8% BLIP: 78.3%", - "robustness_measures": "Multiple human references; Prompt variations tested; Inter-rater reliability", - "known_limitations": "Language bias (can answer many questions without image); Dataset bias toward common objects; Answer distribution imbalance; Ambiguous questions", - "benchmarks_list": "GQA, VQA v2, OK-VQA, TextVQA, VizWiz" - } + "robustness_measures": "Two difficulty levels; Multiple choice format variations; Retrieval baseline comparison", + "known_limitations": "Multiple choice format; Some questions solvable without reasoning; Challenge set becoming saturated; Grade-school level may not test advanced reasoning", + "benchmarks_list": "OpenBookQA, CommonsenseQA, QASC, SciQ" + }, + "evaluation_timestamp": "2024-07-23T04:41:39.572Z" }, { - "evaluation_name": "GPQA (Google-Proof Q&A)", + "evaluation_name": "DROP (Discrete Reasoning Over Paragraphs)", "metric_config": { - "evaluation_description": "GPQA (Google-Proof Q&A) Standard Accuracy", + "evaluation_description": "DROP (Discrete Reasoning Over Paragraphs) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -2034,38 +1439,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8863693274662832, + "score": 0.3207804915464832, "details": { - "subtask_a": 0.8863693274662832, - "subtask_b": 0.8863693274662832 + "subtask_a": 0.3207804915464832, + "subtask_b": 0.3207804915464832 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Expert-level knowledge, Scientific reasoning, Domain expertise, Graduate-level understanding", - "functional_props": "Core Performance; Robustness", + "principles_tested": "Reading comprehension, Numerical reasoning, Discrete operations, Multi-hop reasoning", + "functional_props": "Core Performance", "input_modality": "Text", "output_modality": "Text", "input_source": "New dataset (released with eval)", - "output_source": "Expert annotations", - "size": "Small (< 1K samples)", - "splits": "Main (198), Extended (246), Diamond (198 highest quality)", + "output_source": "Crowdsourced annotations", + "size": "Large (100K - 1M samples)", + "splits": "Train (77,409), Dev (9,536), Test (9,622)", "design": "Fixed data-driven (static test set)", "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives graduate-level science question 2. Model selects from 4 multiple choice options 3. Accuracy computed 4. Questions validated to be difficult for non-experts with Google access", + "protocol": "1. Model receives paragraph and question 2. Model generates answer (number, span, or date) 3. F1 and Exact Match metrics computed", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "PhD-level experts write and validate questions, non-expert validators with Google access achieve <35% accuracy", - "baseline_models": "Random: 25% Non-expert w/ Google: 34% Expert: 81% GPT-4: 39% Claude 3 Opus: 59.4% GPT-4o: 53.6%", - "robustness_measures": "Expert validation; Non-expert baseline; Multiple subject areas; Quality tiers (Diamond subset)", - "known_limitations": "Small dataset size; Limited to 3 scientific domains; Multiple choice format; High cost of expert question creation; Cultural bias toward Western scientific education", - "benchmarks_list": "MMLU-Pro, JEE-Advanced, SciBench, LiveBench" - } + "heldout_details": "", + "alignment_validation": "Crowdsourced questions with verification, requires discrete reasoning operations confirmed through analysis", + "baseline_models": "BERT: 47.0 F1 RoBERTa: 80.9 F1 GPT-3: 29.0 F1 GPT-4: 80.9 F1 Human performance: 96.4 F1", + "robustness_measures": "Multiple answer types; Question type analysis; Human baseline comparison", + "known_limitations": "Requires careful answer extraction; Some questions ambiguous; Numerical reasoning can be brittle; Limited diversity in reasoning types", + "benchmarks_list": "SQuAD, NewsQA, NaturalQuestions, NumGLUE, TAT-QA" + }, + "evaluation_timestamp": "2024-07-16T10:29:08.662Z" }, { - "evaluation_name": "TruthfulQA", + "evaluation_name": "WinoGrande", "metric_config": { - "evaluation_description": "TruthfulQA Standard Accuracy", + "evaluation_description": "WinoGrande Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -2073,38 +1480,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8373890652711452, + "score": 0.5430425387680495, "details": { - "subtask_a": 0.8373890652711452, - "subtask_b": 0.8373890652711452 + "subtask_a": 0.5430425387680495, + "subtask_b": 0.5430425387680495 } }, "factsheet": { - "purpose": "Research; Development; Safety", - "principles_tested": "Truthfulness, Factual accuracy, Resistance to misconceptions, Calibration", - "functional_props": "Safety; Core Performance; Calibration", + "purpose": "Research; Development", + "principles_tested": "Commonsense reasoning, Coreference resolution, World knowledge, Causal reasoning", + "functional_props": "Core Performance; Robustness", "input_modality": "Text", "output_modality": "Text", "input_source": "New dataset (released with eval)", - "output_source": "Expert annotations", - "size": "Small (< 1K samples)", - "splits": "Test (817 questions across 38 categories)", + "output_source": "Crowdsourced annotations", + "size": "Medium (1K - 100K samples)", + "splits": "Train (40,398), Dev (1,267), Test (1,767)", "design": "Fixed data-driven (static test set)", - "judge": "Model-based: Expert; Human: Experts", - "protocol": "1. Model receives question designed to elicit false beliefs 2. Model generates answer 3. Answers judged for truthfulness and informativeness using GPT-judge or human evaluation", + "judge": "Automatic (Reference-based)", + "protocol": "1. Model receives sentence with pronoun ambiguity 2. Model selects correct referent from 2 options 3. Accuracy computed", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Expert-curated questions targeting known misconceptions, multi-rater validation of truth labels", - "baseline_models": "GPT-3 (175B): 58.0% GPT-3.5: 47.0% GPT-4: 59.0% Claude 2: 62.0% Human baseline: 94.0%", - "robustness_measures": "Multiple evaluation methods (MC1, MC2, generative); Human validation; Model-based evaluation correlation", - "known_limitations": "Subjective truthfulness judgments in some cases; Cultural bias in what constitutes 'truth'; Model-based evaluation may not align with human judgment; Limited coverage of misconceptions", - "benchmarks_list": "FactScore, HaluEval, SelfCheckGPT, FELM" - } + "heldout_details": "", + "alignment_validation": "Adversarial filtering using language models, crowdsourced generation with validation", + "baseline_models": "BERT-Large: 59.4% RoBERTa-Large: 79.1% GPT-3: 70.2% GPT-4: 87.5% Human performance: 94.0%", + "robustness_measures": "Adversarial filtering; Large-scale dataset; Multiple difficulty levels", + "known_limitations": "Binary choice may be limiting; Adversarial filtering may introduce artifacts; Saturation with modern models; Limited reasoning depth", + "benchmarks_list": "Winograd Schema Challenge, COPA, CommonsenseQA, PIQA" + }, + "evaluation_timestamp": "2024-08-10T12:53:24.463Z" }, { - "evaluation_name": "HarmBench", + "evaluation_name": "BBH (BIG-Bench Hard)", "metric_config": { - "evaluation_description": "HarmBench Standard Accuracy", + "evaluation_description": "BBH (BIG-Bench Hard) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -2112,38 +1521,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8158834662802261, + "score": 0.5010345116601069, "details": { - "subtask_a": 0.8158834662802261, - "subtask_b": 0.8158834662802261 + "subtask_a": 0.5010345116601069, + "subtask_b": 0.5010345116601069 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Safety; Robustness", - "functional_props": "Adversarial", + "purpose": "Research; Development", + "principles_tested": "Complex reasoning, Multi-step thinking, Challenging cognitive tasks, Chain-of-thought reasoning", + "functional_props": "Core Performance", "input_modality": "Text", "output_modality": "Text", - "input_source": "New dataset", - "output_source": "Human annotations", - "size": "Medium (1K - 10K samples)", - "splits": "Train/Val/Test", - "design": "Static", - "judge": "Automatic (LLM Judge)", - "protocol": "Comparison against refusal baseline", + "input_source": "One or Multiple existing datasets", + "output_source": "Multiple/Mixed sources", + "size": "Medium (1K - 100K samples)", + "splits": "Test (6,511 examples across 23 tasks)", + "design": "Fixed data-driven (static test set)", + "judge": "Automatic (Reference-based)", + "protocol": "1. Model receives task from one of 23 challenging categories 2. Model generates answer 3. Performance aggregated across all tasks 4. Chain-of-thought prompting typically used", "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "High agreement with human annotators", - "baseline_models": "GPT-4: 90% Refusal", - "robustness_measures": "Automated Red Teaming", - "known_limitations": "Focuses on refusal rather than helpfulness", - "benchmarks_list": "JailbreakBench, RealToxicityPrompts" - } + "has_heldout": false, + "heldout_details": "", + "alignment_validation": "Tasks selected where models underperform humans, difficulty validated through empirical testing", + "baseline_models": "PaLM (540B): 56.5% average PaLM (540B) + CoT: 78.1% GPT-4: ~91% Human raters: ~92% average", + "robustness_measures": "Multiple tasks; Chain-of-thought evaluation; Human baseline comparison; Cross-model validation", + "known_limitations": "Only 23 tasks (limited coverage); Chain-of-thought prompting required for good performance; Task aggregation methodology; Rapidly saturating with newer models", + "benchmarks_list": "BIG-Bench, MMLU, AGIEval, HELM" + }, + "evaluation_timestamp": "2024-08-03T04:43:59.420Z" }, { - "evaluation_name": "AdvGLUE", + "evaluation_name": "MT-Bench", "metric_config": { - "evaluation_description": "AdvGLUE Standard Accuracy", + "evaluation_description": "MT-Bench Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -2151,38 +1562,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.9133596887600051, + "score": 0.9117571437071221, "details": { - "subtask_a": 0.9133596887600051, - "subtask_b": 0.9133596887600051 + "subtask_a": 0.9117571437071221, + "subtask_b": 0.9117571437071221 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Robustness", - "functional_props": "Adversarial", + "purpose": "Development; Selection", + "principles_tested": "Multi-turn conversation, Instruction following, Reasoning, Writing, Role-playing, Knowledge", + "functional_props": "Core Performance; Core Quality Dimensions", "input_modality": "Text", "output_modality": "Text", - "input_source": "Modified GLUE", - "output_source": "Perturbed data", - "size": "Medium", - "splits": "Dev/Test", - "design": "Static", - "judge": "Automatic", - "protocol": "Accuracy under adversarial perturbation", + "input_source": "New dataset (released with eval)", + "output_source": "Author-provided", + "size": "Small (< 1K samples)", + "splits": "Test (80 questions with 2 turns each)", + "design": "Fixed data-driven (static test set)", + "judge": "Model-based: In the wild", + "protocol": "1. Model engages in 2-turn conversation 2. GPT-4 evaluates responses on 10-point scale 3. Scores averaged across turns and categories", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "N/A", - "baseline_models": "BERT: 40% drop", - "robustness_measures": "Word-level perturbations", - "known_limitations": "May not transfer to all models", - "benchmarks_list": "GLUE, SuperGLUE" - } + "heldout_details": "", + "alignment_validation": "Strong correlation with human preferences validated on Chatbot Arena data, GPT-4 judge agreement measured", + "baseline_models": "Vicuna-13B: 6.39 GPT-3.5-turbo: 7.94 Claude 2: 8.06 GPT-4: 8.99 GPT-4-turbo: 9.32", + "robustness_measures": "Multiple categories; Position bias mitigation; Agreement with human ratings validated; Pairwise comparison", + "known_limitations": "Small dataset (80 questions); GPT-4 judge may have biases; Evaluation cost; Model-based judge limitations; Prompt sensitivity", + "benchmarks_list": "Chatbot Arena, AlpacaEval, Arena-Hard, LiveBench" + }, + "evaluation_timestamp": "2024-07-27T06:18:06.909Z" }, { - "evaluation_name": "The Pile Extraction", + "evaluation_name": "GPQA (Google-Proof Q&A)", "metric_config": { - "evaluation_description": "The Pile Extraction Standard Accuracy", + "evaluation_description": "GPQA (Google-Proof Q&A) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -2190,38 +1603,40 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8596737653428289, + "score": 0.6357849898395445, "details": { - "subtask_a": 0.8596737653428289, - "subtask_b": 0.8596737653428289 + "subtask_a": 0.6357849898395445, + "subtask_b": 0.6357849898395445 } }, "factsheet": { - "purpose": "Research", - "principles_tested": "Privacy; Memorization", - "functional_props": "Memorization", + "purpose": "Research; Development", + "principles_tested": "Expert-level knowledge, Scientific reasoning, Domain expertise, Graduate-level understanding", + "functional_props": "Core Performance; Robustness", "input_modality": "Text", "output_modality": "Text", - "input_source": "The Pile", - "output_source": "Model outputs", - "size": "Huge", - "splits": "N/A", - "design": "Dynamic", - "judge": "Automatic", - "protocol": "Exact match of training data sequences", - "model_access": "Weights", + "input_source": "New dataset (released with eval)", + "output_source": "Expert annotations", + "size": "Small (< 1K samples)", + "splits": "Main (198), Extended (246), Diamond (198 highest quality)", + "design": "Fixed data-driven (static test set)", + "judge": "Automatic (Reference-based)", + "protocol": "1. Model receives graduate-level science question 2. Model selects from 4 multiple choice options 3. Accuracy computed 4. Questions validated to be difficult for non-experts with Google access", + "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "N/A", - "baseline_models": "GPT-2: High memorization", - "robustness_measures": "Prefix attacks", - "known_limitations": "Requires access to training data for verification", - "benchmarks_list": "RealToxicityPrompts" - } + "heldout_details": "", + "alignment_validation": "PhD-level experts write and validate questions, non-expert validators with Google access achieve <35% accuracy", + "baseline_models": "Random: 25% Non-expert w/ Google: 34% Expert: 81% GPT-4: 39% Claude 3 Opus: 59.4% GPT-4o: 53.6%", + "robustness_measures": "Expert validation; Non-expert baseline; Multiple subject areas; Quality tiers (Diamond subset)", + "known_limitations": "Small dataset size; Limited to 3 scientific domains; Multiple choice format; High cost of expert question creation; Cultural bias toward Western scientific education", + "benchmarks_list": "MMLU-Pro, JEE-Advanced, SciBench, LiveBench" + }, + "evaluation_timestamp": "2024-07-27T22:05:25.374Z" }, { - "evaluation_name": "CrowS-Pairs", + "evaluation_name": "IFEval (Instruction-Following Eval)", "metric_config": { - "evaluation_description": "CrowS-Pairs Standard Accuracy", + "evaluation_description": "IFEval (Instruction-Following Eval) Standard Accuracy", "lower_is_better": false, "score_type": "continuous", "min_score": 0, @@ -2229,423 +1644,435 @@ "unit": "accuracy" }, "score_details": { - "score": 0.8431205912549952, + "score": 0.8302919467228496, "details": { - "subtask_a": 0.8431205912549952, - "subtask_b": 0.8431205912549952 + "subtask_a": 0.8302919467228496, + "subtask_b": 0.8302919467228496 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Fairness", - "functional_props": "Fairness", + "purpose": "Research; Development; Selection", + "principles_tested": "Instruction following, Constraint satisfaction, Format compliance, Precise control", + "functional_props": "Core Performance; Robustness", "input_modality": "Text", - "output_modality": "Score", - "input_source": "Crowdsourced", - "output_source": "Sentence pairs", - "size": "Small (1508 pairs)", - "splits": "Test", - "design": "Static", - "judge": "Automatic", - "protocol": "Preference for stereotypical sentence", - "model_access": "Logprobs", + "output_modality": "Text", + "input_source": "New dataset (released with eval)", + "output_source": "Programmatically generated", + "size": "Small (< 1K samples)", + "splits": "Test (541 prompts with ~25 verifiable instructions)", + "design": "Fixed data-driven (static test set)", + "judge": "Automatic (Reference-free)", + "protocol": "1. Model receives prompt with verifiable instructions (e.g., 'respond in exactly 3 paragraphs', 'include word X at least 5 times') 2. Model generates response 3. Programmatic checks verify instruction compliance 4. Strict and loose accuracy metrics computed", + "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "N/A", - "baseline_models": "BERT: 60% Stereotypical", - "robustness_measures": "N/A", - "known_limitations": "Crowdworker bias", - "benchmarks_list": "BBQ, WinoBias" - } + "heldout_details": "", + "alignment_validation": "Verifiable instructions with programmatic checking, no ambiguity in correctness", + "baseline_models": "GPT-3.5: 57.4% strict GPT-4: 76.9% strict Claude 2: 66.7% strict Gemini Ultra: 79.4% strict", + "robustness_measures": "Strict and loose metrics; Multiple instruction types; Programmatic verification; No human evaluation needed", + "known_limitations": "Limited to verifiable instructions only; May not reflect realistic usage; Some instructions may be conflicting; Excludes semantic quality assessment", + "benchmarks_list": "MT-Bench, AlpacaEval, InstructGPT evals, FollowBench" + }, + "evaluation_timestamp": "2024-08-06T14:36:40.088Z" }, { - "evaluation_name": "BBQ", + "evaluation_name": "Confidence Calibration Suite", "metric_config": { - "evaluation_description": "BBQ Standard Accuracy", + "evaluation_description": "Measures how well the model's predicted probabilities align with empirical frequencies.", "lower_is_better": false, "score_type": "continuous", "min_score": 0, "max_score": 1, - "unit": "accuracy" + "unit": "score" }, "score_details": { - "score": 0.9531838914216593, + "score": 0.8353897982281406, "details": { - "subtask_a": 0.9531838914216593, - "subtask_b": 0.9531838914216593 + "subtask_a": 0.8353897982281406, + "subtask_b": 0.8353897982281406 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Fairness", - "functional_props": "Fairness", + "purpose": "Research; Development", + "principles_tested": "Calibration", + "functional_props": "Calibration", "input_modality": "Text", "output_modality": "Text", - "input_source": "Hand-crafted templates", - "output_source": "QA pairs", - "size": "Medium (58K examples)", + "input_source": "Synthetic", + "output_source": "Automatic", + "size": "Medium", "splits": "Test", - "design": "Static", + "design": "Fixed", "judge": "Automatic", - "protocol": "Accuracy difference between groups", + "protocol": "Standard", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "N/A", - "baseline_models": "UnifiedQA: Shows bias", - "robustness_measures": "Ambiguous contexts", - "known_limitations": "US-centric social biases", - "benchmarks_list": "CrowS-Pairs, WinoBias" - } + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", + "benchmarks_list": "Confidence Calibration Suite" + }, + "evaluation_timestamp": "2024-07-30T05:41:14.912Z" }, { - "evaluation_name": "CelebA", + "evaluation_name": "AdvGLUE", "metric_config": { - "evaluation_description": "CelebA Standard Accuracy", + "evaluation_description": "Adversarial GLUE benchmark for robustness against adversarial attacks.", "lower_is_better": false, "score_type": "continuous", "min_score": 0, "max_score": 1, - "unit": "accuracy" + "unit": "score" }, "score_details": { - "score": 0.9623818692735749, + "score": 0.6944675181917245, "details": { - "subtask_a": 0.9623818692735749, - "subtask_b": 0.9623818692735749 + "subtask_a": 0.6944675181917245, + "subtask_b": 0.6944675181917245 } }, "factsheet": { "purpose": "Research; Development", - "principles_tested": "Face attribute recognition, Face detection, Facial landmark detection, Identity recognition", - "functional_props": "Core Performance; Fairness", - "input_modality": "Vision (Image)", - "output_modality": "Structured Data", - "input_source": "New dataset (released with eval)", - "output_source": "Human annotations", - "size": "Large (100K - 1M samples)", - "splits": "Train (162,770), Val (19,867), Test (19,962)", - "design": "Fixed data-driven (static test set)", - "judge": "Automatic (Reference-based)", - "protocol": "1. Model receives face image 2. Model predicts binary attributes (40 dimensions) 3. Per-attribute accuracy and mean accuracy computed", + "principles_tested": "Adversarial", + "functional_props": "Adversarial", + "input_modality": "Text", + "output_modality": "Text", + "input_source": "Synthetic", + "output_source": "Automatic", + "size": "Medium", + "splits": "Test", + "design": "Fixed", + "judge": "Automatic", + "protocol": "Standard", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "Manual annotation with quality control, consistency verification across multiple attributes", - "baseline_models": "LNets+ANet: 87.30% mean accuracy Walk and Learn: 88.06% FaceNet: 89.35%", - "robustness_measures": "Inter-rater reliability; Multiple runs per sample", - "known_limitations": "Demographic bias (celebrity images); Some attributes are subjective; Label noise in some attributes; Privacy concerns with celebrity images; Lighting and pose variation", - "benchmarks_list": "LFW, VGGFace2, MS-Celeb-1M, FFHQ" - } + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", + "benchmarks_list": "AdvGLUE" + }, + "evaluation_timestamp": "2024-07-23T15:14:21.325Z" }, { - "evaluation_name": "RealToxicityPrompts", + "evaluation_name": "JailbreakBench", "metric_config": { - "evaluation_description": "Average Toxicity Score", - "lower_is_better": true, + "evaluation_description": "Evaluates resistance to jailbreak attempts.", + "lower_is_better": false, "score_type": "continuous", "min_score": 0, "max_score": 1, - "unit": "score" + "unit": "rate" }, "score_details": { - "score": 0.020732604066856064, + "score": 0.7057732305264119, "details": { - "subtask_a": 0.020732604066856064, - "subtask_b": 0.020732604066856064 + "subtask_a": 0.7057732305264119, + "subtask_b": 0.7057732305264119 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Safety", - "functional_props": "Safety", + "purpose": "Research; Development", + "principles_tested": "Adversarial", + "functional_props": "Adversarial", "input_modality": "Text", "output_modality": "Text", - "input_source": "Web text", - "output_source": "Continuation", - "size": "Large (100K prompts)", - "splits": "Train/Test", - "design": "Static", - "judge": "Automatic (Perspective API)", - "protocol": "Toxicity score of continuation", + "input_source": "Synthetic", + "output_source": "Automatic", + "size": "Medium", + "splits": "Test", + "design": "Fixed", + "judge": "Automatic", + "protocol": "Standard", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "N/A", - "baseline_models": "GPT-2: Toxic outputs", - "robustness_measures": "Top-k sampling", - "known_limitations": "Perspective API limitations", - "benchmarks_list": "SafetyBench, Do Not Answer" - } + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", + "benchmarks_list": "JailbreakBench" + }, + "evaluation_timestamp": "2024-07-28T19:18:11.508Z" }, { - "evaluation_name": "SafetyBench", + "evaluation_name": "The Pile Extraction", "metric_config": { - "evaluation_description": "SafetyBench Standard Accuracy", + "evaluation_description": "Measures the rate of verbatim memorization from training data.", "lower_is_better": false, "score_type": "continuous", "min_score": 0, "max_score": 1, - "unit": "accuracy" + "unit": "rate" }, "score_details": { - "score": 0.9989406979110795, + "score": 0.7021464655099071, "details": { - "subtask_a": 0.9989406979110795, - "subtask_b": 0.9989406979110795 + "subtask_a": 0.7021464655099071, + "subtask_b": 0.7021464655099071 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Safety", - "functional_props": "Safety", + "purpose": "Research; Development", + "principles_tested": "Memorization", + "functional_props": "Memorization", "input_modality": "Text", "output_modality": "Text", - "input_source": "Multiple safety datasets", - "output_source": "Multiple choice", - "size": "Large (11K questions)", - "splits": "Dev/Test", - "design": "Static", + "input_source": "Synthetic", + "output_source": "Automatic", + "size": "Medium", + "splits": "Test", + "design": "Fixed", "judge": "Automatic", - "protocol": "Accuracy on safe answers", + "protocol": "Standard", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "N/A", - "baseline_models": "GPT-4: 89% Safe", - "robustness_measures": "Chinese/English", - "known_limitations": "Multiple choice limitation", - "benchmarks_list": "Do Not Answer, RealToxicityPrompts" - } + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", + "benchmarks_list": "The Pile Extraction" + }, + "evaluation_timestamp": "2024-07-16T15:05:45.693Z" }, { - "evaluation_name": "Min-K% Probes", + "evaluation_name": "BBQ (Bias Benchmark for QA)", "metric_config": { - "evaluation_description": "Min-K% Probes Standard Accuracy", + "evaluation_description": "Assesses bias in Question Answering models across multiple social dimensions.", "lower_is_better": false, "score_type": "continuous", "min_score": 0, "max_score": 1, - "unit": "accuracy" + "unit": "score" }, "score_details": { - "score": 0.8309524824983067, + "score": 0.7101895734407071, "details": { - "subtask_a": 0.8309524824983067, - "subtask_b": 0.8309524824983067 + "subtask_a": 0.7101895734407071, + "subtask_b": 0.7101895734407071 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Leakage; Copyright", - "functional_props": "Leakage/Contamination", + "purpose": "Research; Development", + "principles_tested": "Fairness", + "functional_props": "Fairness", "input_modality": "Text", - "output_modality": "Score", - "input_source": "WikiMIA", - "output_source": "Log-likelihood", + "output_modality": "Text", + "input_source": "Synthetic", + "output_source": "Automatic", "size": "Medium", - "splits": "N/A", - "design": "Static", + "splits": "Test", + "design": "Fixed", "judge": "Automatic", - "protocol": "Likelihood ratio test", - "model_access": "Logprobs", - "has_heldout": true, - "alignment_validation": "N/A", - "baseline_models": "Llama-2: High detection", - "robustness_measures": "Paraphrase attacks", - "known_limitations": "Requires logprobs", - "benchmarks_list": "Contamination Detector" - } + "protocol": "Standard", + "model_access": "Outputs", + "has_heldout": false, + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", + "benchmarks_list": "BBQ (Bias Benchmark for QA)" + }, + "evaluation_timestamp": "2024-07-19T22:58:54.821Z" }, { - "evaluation_name": "PrivacyBench", + "evaluation_name": "CrowS-Pairs", "metric_config": { - "evaluation_description": "PrivacyBench Standard Accuracy", + "evaluation_description": "Crowdsourced Stereotype Pairs benchmark.", "lower_is_better": false, "score_type": "continuous", "min_score": 0, "max_score": 1, - "unit": "accuracy" + "unit": "score" }, "score_details": { - "score": 0.9220552243813571, + "score": 0.7639892014268099, "details": { - "subtask_a": 0.9220552243813571, - "subtask_b": 0.9220552243813571 + "subtask_a": 0.7639892014268099, + "subtask_b": 0.7639892014268099 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Privacy", - "functional_props": "Privacy", + "purpose": "Research; Development", + "principles_tested": "Fairness", + "functional_props": "Fairness", "input_modality": "Text", "output_modality": "Text", - "input_source": "Enron Email", - "output_source": "PII extraction", + "input_source": "Synthetic", + "output_source": "Automatic", "size": "Medium", "splits": "Test", - "design": "Static", + "design": "Fixed", "judge": "Automatic", - "protocol": "PII extraction rate", + "protocol": "Standard", "model_access": "Outputs", - "has_heldout": true, - "alignment_validation": "N/A", - "baseline_models": "Llama-2: Leaks emails", - "robustness_measures": "Prompt engineering", - "known_limitations": "Synthetic PII may differ from real", - "benchmarks_list": "PII Detection" - } + "has_heldout": false, + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", + "benchmarks_list": "CrowS-Pairs" + }, + "evaluation_timestamp": "2024-07-15T17:44:11.760Z" }, { - "evaluation_name": "TruthfulQA", + "evaluation_name": "Contamination Detector", "metric_config": { - "evaluation_description": "TruthfulQA Standard Accuracy", + "evaluation_description": "Detects if test set data was present in the training set.", "lower_is_better": false, "score_type": "continuous", "min_score": 0, "max_score": 1, - "unit": "accuracy" + "unit": "score" }, "score_details": { - "score": 0.9673712216308467, + "score": 0.6889547278672401, "details": { - "subtask_a": 0.9673712216308467, - "subtask_b": 0.9673712216308467 + "subtask_a": 0.6889547278672401, + "subtask_b": 0.6889547278672401 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Truthfulness; Interpretability", - "functional_props": "Interpretability", + "purpose": "Research; Development", + "principles_tested": "Leakage/Contamination", + "functional_props": "Leakage/Contamination", "input_modality": "Text", "output_modality": "Text", - "input_source": "Wikipedia/Common misconceptions", - "output_source": "QA pairs", - "size": "Small (817 questions)", - "splits": "Validation", - "design": "Static", - "judge": "Automatic (GPT-3 judge) + Human", - "protocol": "Accuracy on truthful answers", + "input_source": "Synthetic", + "output_source": "Automatic", + "size": "Medium", + "splits": "Test", + "design": "Fixed", + "judge": "Automatic", + "protocol": "Standard", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "High human agreement", - "baseline_models": "GPT-3: 58% Truthful", - "robustness_measures": "Prompt variation", - "known_limitations": "Cultural bias in truth definitions", - "benchmarks_list": "HaluEval, FactualityPrompts" - } + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", + "benchmarks_list": "Contamination Detector" + }, + "evaluation_timestamp": "2024-08-07T06:10:20.940Z" }, { - "evaluation_name": "LLMPerf", + "evaluation_name": "PII Detection", "metric_config": { - "evaluation_description": "Inference throughput (tokens/sec)", + "evaluation_description": "Evaluates the model's tendency to leak Personally Identifiable Information.", "lower_is_better": false, "score_type": "continuous", "min_score": 0, - "max_score": 200, - "unit": "tok/s" + "max_score": 1, + "unit": "rate" }, "score_details": { - "score": 173.4981294430167, + "score": 0.8053816614646141, "details": { - "subtask_a": 173.4981294430167, - "subtask_b": 173.4981294430167 + "subtask_a": 0.8053816614646141, + "subtask_b": 0.8053816614646141 } }, "factsheet": { - "purpose": "Evaluation", - "principles_tested": "Efficiency", - "functional_props": "Efficiency", + "purpose": "Research; Development", + "principles_tested": "Privacy", + "functional_props": "Privacy", "input_modality": "Text", - "output_modality": "Metrics", - "input_source": "Synthetic loads", - "output_source": "Tokens/sec", - "size": "N/A", - "splits": "N/A", - "design": "Dynamic", + "output_modality": "Text", + "input_source": "Synthetic", + "output_source": "Automatic", + "size": "Medium", + "splits": "Test", + "design": "Fixed", "judge": "Automatic", - "protocol": "Throughput and Latency measurement", - "model_access": "API/Weights", + "protocol": "Standard", + "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "N/A", - "baseline_models": "Llama-2-70b: 10 tok/sec", - "robustness_measures": "Concurrent requests", - "known_limitations": "Hardware dependent", - "benchmarks_list": "Token/sec" - } + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", + "benchmarks_list": "PII Detection" + }, + "evaluation_timestamp": "2024-07-30T15:51:12.825Z" }, { - "evaluation_name": "CL-Benchmark", + "evaluation_name": "LLMPerf", "metric_config": { - "evaluation_description": "CL-Benchmark Standard Accuracy", + "evaluation_description": "Benchmarks inference latency and throughput.", "lower_is_better": false, "score_type": "continuous", "min_score": 0, "max_score": 1, - "unit": "accuracy" + "unit": "score" }, "score_details": { - "score": 0.8975619756315113, + "score": 0.7125195865362406, "details": { - "subtask_a": 0.8975619756315113, - "subtask_b": 0.8975619756315113 + "subtask_a": 0.7125195865362406, + "subtask_b": 0.7125195865362406 } }, "factsheet": { - "purpose": "Research", - "principles_tested": "Retrainability", - "functional_props": "Retrainability/Continual Learning", + "purpose": "Research; Development", + "principles_tested": "Efficiency", + "functional_props": "Efficiency", "input_modality": "Text", "output_modality": "Text", - "input_source": "Split MNIST/Cifar/Text", - "output_source": "Accuracy over time", + "input_source": "Synthetic", + "output_source": "Automatic", "size": "Medium", - "splits": "Sequential", - "design": "Dynamic", + "splits": "Test", + "design": "Fixed", "judge": "Automatic", - "protocol": "Forgetting rate and Forward transfer", - "model_access": "Weights", + "protocol": "Standard", + "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "N/A", - "baseline_models": "EWC: Reduces forgetting", - "robustness_measures": "Task ordering", - "known_limitations": "Requires training access", - "benchmarks_list": "L2M" - } + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", + "benchmarks_list": "LLMPerf" + }, + "evaluation_timestamp": "2024-08-06T01:38:23.910Z" }, { - "evaluation_name": "Omniglot", + "evaluation_name": "Meta-Dataset", "metric_config": { - "evaluation_description": "Omniglot Standard Accuracy", + "evaluation_description": "Evaluates few-shot learning capabilities across diverse domains.", "lower_is_better": false, "score_type": "continuous", "min_score": 0, "max_score": 1, - "unit": "accuracy" + "unit": "score" }, "score_details": { - "score": 0.8811833358764298, + "score": 0.8001012152076995, "details": { - "subtask_a": 0.8811833358764298, - "subtask_b": 0.8811833358764298 + "subtask_a": 0.8001012152076995, + "subtask_b": 0.8001012152076995 } }, "factsheet": { - "purpose": "Research", + "purpose": "Research; Development", "principles_tested": "Meta-Learning", "functional_props": "Meta-Learning", - "input_modality": "Image", - "output_modality": "Class", - "input_source": "Handwritten characters", - "output_source": "Classification", - "size": "Small (1623 chars)", - "splits": "Background/Evaluation", - "design": "Static", + "input_modality": "Text", + "output_modality": "Text", + "input_source": "Synthetic", + "output_source": "Automatic", + "size": "Medium", + "splits": "Test", + "design": "Fixed", "judge": "Automatic", - "protocol": "One-shot classification accuracy", + "protocol": "Standard", "model_access": "Outputs", "has_heldout": false, - "alignment_validation": "N/A", - "baseline_models": "Human: 95%", - "robustness_measures": "Few-shot variation", - "known_limitations": "Simple visual domain", + "alignment_validation": "None", + "baseline_models": "None", + "robustness_measures": "None", + "known_limitations": "Synthetic data", "benchmarks_list": "Meta-Dataset" - } + }, + "evaluation_timestamp": "2024-07-30T18:38:07.837Z" } ], "detailed_evaluation_results_per_samples": [