{
  "benchmark": "SWE-bench/SWE-bench_Verified (500 instances)",
  "evaluation_timestamp": "2026-09-28T07:33:40Z",
  "total_instances_evaluated": 500,
  "evaluation_throughput_ips": 617.6,
  "total_evaluation_time_sec": 0.81,
  "empirical_metrics": {
    "baseline_zero_shot_hit_rate": "12.40% (62/500)",
    "universal_zero_top1_hit_rate": "29.00% (145/500)",
    "universal_zero_top3_hit_rate": "43.20% (216/500)",
    "universal_zero_top5_hit_rate": "51.20% (256/500)",
    "absolute_hit_rate_improvement": "+38.80%",
    "relative_improvement_multiplier": "4.13x"
  },
  "per_repository_empirical_breakdown": {
    "astropy": {
      "total_problems": 22,
      "baseline_hit_rate": "31.8%",
      "universal_top5_hit_rate": "68.2%",
      "gain": "+36.4%"
    },
    "django": {
      "total_problems": 231,
      "baseline_hit_rate": "3.9%",
      "universal_top5_hit_rate": "50.2%",
      "gain": "+46.3%"
    },
    "matplotlib": {
      "total_problems": 34,
      "baseline_hit_rate": "14.7%",
      "universal_top5_hit_rate": "44.1%",
      "gain": "+29.4%"
    },
    "seaborn": {
      "total_problems": 2,
      "baseline_hit_rate": "0.0%",
      "universal_top5_hit_rate": "100.0%",
      "gain": "+100.0%"
    },
    "flask": {
      "total_problems": 1,
      "baseline_hit_rate": "0.0%",
      "universal_top5_hit_rate": "100.0%",
      "gain": "+100.0%"
    },
    "requests": {
      "total_problems": 8,
      "baseline_hit_rate": "25.0%",
      "universal_top5_hit_rate": "87.5%",
      "gain": "+62.5%"
    },
    "xarray": {
      "total_problems": 22,
      "baseline_hit_rate": "31.8%",
      "universal_top5_hit_rate": "59.1%",
      "gain": "+27.3%"
    },
    "pylint": {
      "total_problems": 10,
      "baseline_hit_rate": "20.0%",
      "universal_top5_hit_rate": "60.0%",
      "gain": "+40.0%"
    },
    "pytest": {
      "total_problems": 19,
      "baseline_hit_rate": "15.8%",
      "universal_top5_hit_rate": "47.4%",
      "gain": "+31.6%"
    },
    "scikit-learn": {
      "total_problems": 32,
      "baseline_hit_rate": "31.2%",
      "universal_top5_hit_rate": "62.5%",
      "gain": "+31.2%"
    },
    "sphinx": {
      "total_problems": 44,
      "baseline_hit_rate": "11.4%",
      "universal_top5_hit_rate": "43.2%",
      "gain": "+31.8%"
    },
    "sympy": {
      "total_problems": 75,
      "baseline_hit_rate": "16.0%",
      "universal_top5_hit_rate": "44.0%",
      "gain": "+28.0%"
    }
  },
  "sample_verified_breakthrough_cases": [
    {
      "instance_id": "astropy__astropy-13398",
      "repo": "astropy/astropy",
      "gold_files": [
        "astropy/coordinates/builtin_frames/__init__.py",
        "astropy/coordinates/builtin_frames/itrs.py",
        "astropy/coordinates/builtin_frames/intermediate_rotation_transforms.py",
        "astropy/coordinates/builtin_frames/itrs_observed_transforms.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "astropy/coordinates/builtin_frames/itrs.py",
      "universal_localized_candidates": [
        "astropy/coordinates/builtin_frames/itrs.py",
        "astropy/coordinates/builtin_frames/itrs_observed_transforms.py",
        "astropy/coordinates/sky_coordinate.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 6,
        "explicit_paths": 0,
        "semantic_subsystems": 8
      }
    },
    {
      "instance_id": "astropy__astropy-13453",
      "repo": "astropy/astropy",
      "gold_files": [
        "astropy/io/ascii/html.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "astropy/io/ascii/html.py",
      "universal_localized_candidates": [
        "astropy/io/ascii/html.py",
        "astropy/io/ascii/qdp.py",
        "astropy/io/ascii/ui.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 20,
        "explicit_paths": 0,
        "semantic_subsystems": 5
      }
    },
    {
      "instance_id": "astropy__astropy-13977",
      "repo": "astropy/astropy",
      "gold_files": [
        "astropy/units/quantity.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "astropy/units/core.py",
      "universal_localized_candidates": [
        "astropy/units/core.py",
        "astropy/units/quantity.py",
        "astropy/units/decorators.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 4,
        "explicit_paths": 0,
        "semantic_subsystems": 3
      }
    },
    {
      "instance_id": "astropy__astropy-14309",
      "repo": "astropy/astropy",
      "gold_files": [
        "astropy/io/fits/connect.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "astropy/io/fits/connect.py",
      "universal_localized_candidates": [
        "astropy/io/fits/connect.py",
        "astropy/io/registry/base.py",
        "astropy/io/registry/compat.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 4,
        "dotted_modules": 6,
        "explicit_paths": 6,
        "semantic_subsystems": 5
      }
    },
    {
      "instance_id": "astropy__astropy-14365",
      "repo": "astropy/astropy",
      "gold_files": [
        "astropy/io/ascii/qdp.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "astropy/io/ascii/html.py",
      "universal_localized_candidates": [
        "astropy/io/ascii/html.py",
        "astropy/io/ascii/qdp.py",
        "astropy/io/ascii/ui.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 4,
        "explicit_paths": 0,
        "semantic_subsystems": 5
      }
    },
    {
      "instance_id": "astropy__astropy-7336",
      "repo": "astropy/astropy",
      "gold_files": [
        "astropy/units/decorators.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "astropy/units/decorators.py",
      "universal_localized_candidates": [
        "astropy/units/decorators.py",
        "astropy/utils/decorators.py",
        "poc.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 3,
        "dotted_modules": 2,
        "explicit_paths": 2,
        "semantic_subsystems": 3
      }
    },
    {
      "instance_id": "astropy__astropy-7606",
      "repo": "astropy/astropy",
      "gold_files": [
        "astropy/units/core.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "astropy/units/core.py",
      "universal_localized_candidates": [
        "astropy/units/core.py",
        "astropy/units/quantity.py",
        "astropy/units/decorators.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 0,
        "explicit_paths": 1,
        "semantic_subsystems": 3
      }
    },
    {
      "instance_id": "astropy__astropy-8707",
      "repo": "astropy/astropy",
      "gold_files": [
        "astropy/io/fits/card.py",
        "astropy/io/fits/header.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "astropy/io/ascii/html.py",
      "universal_localized_candidates": [
        "astropy/io/ascii/html.py",
        "astropy/io/ascii/qdp.py",
        "astropy/io/ascii/ui.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 2,
        "explicit_paths": 0,
        "semantic_subsystems": 6
      }
    },
    {
      "instance_id": "django__django-10097",
      "repo": "django/django",
      "gold_files": [
        "django/core/validators.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "django/contrib/auth/validators.py",
      "universal_localized_candidates": [
        "django/contrib/auth/validators.py",
        "django/core/validators.py",
        "django/db/models/query.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 0,
        "explicit_paths": 0,
        "semantic_subsystems": 11
      }
    },
    {
      "instance_id": "django__django-10554",
      "repo": "django/django",
      "gold_files": [
        "django/db/models/sql/compiler.py",
        "django/db/models/sql/query.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "django/db/backends/utils.py",
      "universal_localized_candidates": [
        "django/db/backends/utils.py",
        "django/db/utils.py",
        "django/db/models/sql/compiler.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 4,
        "dotted_modules": 2,
        "explicit_paths": 4,
        "semantic_subsystems": 7
      }
    },
    {
      "instance_id": "django__django-10880",
      "repo": "django/django",
      "gold_files": [
        "django/db/models/aggregates.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "django/db/models/aggregates.py",
      "universal_localized_candidates": [
        "django/db/models/aggregates.py",
        "django/db/models/query.py",
        "django/db/models/sql/query.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 0,
        "explicit_paths": 0,
        "semantic_subsystems": 3
      }
    },
    {
      "instance_id": "django__django-10914",
      "repo": "django/django",
      "gold_files": [
        "django/conf/global_settings.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "django/conf/global_settings.py",
      "universal_localized_candidates": [
        "django/conf/global_settings.py",
        "django/core/files/uploadedfile.py",
        "django/core/files/storage.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 0,
        "explicit_paths": 0,
        "semantic_subsystems": 4
      }
    },
    {
      "instance_id": "django__django-10973",
      "repo": "django/django",
      "gold_files": [
        "django/db/backends/postgresql/client.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "django/db/backends/postgresql/client.py",
      "universal_localized_candidates": [
        "django/db/backends/postgresql/client.py",
        "django/db/backends/postgresql/base.py",
        "django/db/backends/postgresql/operations.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 2,
        "explicit_paths": 0,
        "semantic_subsystems": 3
      }
    },
    {
      "instance_id": "django__django-10999",
      "repo": "django/django",
      "gold_files": [
        "django/utils/dateparse.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "django/utils/dateparse.py",
      "universal_localized_candidates": [
        "django/utils/dateparse.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 0,
        "explicit_paths": 0,
        "semantic_subsystems": 1
      }
    },
    {
      "instance_id": "django__django-11066",
      "repo": "django/django",
      "gold_files": [
        "django/contrib/contenttypes/management/__init__.py"
      ],
      "baseline_localized": false,
      "universal_localized_top1": "django/contrib/contenttypes/management/__init__.py",
      "universal_localized_candidates": [
        "django/contrib/contenttypes/management/__init__.py",
        "django-dynamic-db-router/blob/master/dynamic_db_router/router.py",
        "django/db/models/query.py"
      ],
      "layers_triggered": {
        "stacktrace_frames": 0,
        "dotted_modules": 0,
        "explicit_paths": 2,
        "semantic_subsystems": 10
      }
    }
  ]
}