Spaces:
Running
Running
| [ | |
| { | |
| "space": "healthbench-consensus-medgemma", | |
| "file": "2026-08-05T08-51-22-00-00_healthbench-consensus_6a8D2h492in82NNQHCFgZw.eval", | |
| "task": "healthbench_consensus", | |
| "task_version": 2, | |
| "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "judge_model": "openai/gpt-4.1", | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.002 | |
| }, | |
| "dataset_samples": 3671, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 183, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-08-05T08:51:22+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.252" | |
| }, | |
| "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", | |
| "completed_samples": 3671, | |
| "total_samples": 3671, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.9105, | |
| "_scorer/std": 0.0032, | |
| "_scorer/criteria_met_rate": 0.9066, | |
| "healthbench_score/mean": 0.9105, | |
| "healthbench_score_length_adjusted/mean": 0.9007 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-27b-text-it", | |
| "openai/gpt-4.1" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-27b-text-it": 3662191, | |
| "openai/gpt-4.1": 14399926 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-consensus-medgemma", | |
| "file": "2026-08-05T10-32-09-00-00_healthbench-consensus_6WXSuGaZvDNS5Zze6Jrm3h.eval", | |
| "task": "healthbench_consensus", | |
| "task_version": 2, | |
| "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", | |
| "model": "vllm/google/medgemma-4b-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "judge_model": "openai/gpt-4.1", | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.002 | |
| }, | |
| "dataset_samples": 3671, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 183, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-08-05T10:32:09+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.252" | |
| }, | |
| "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", | |
| "completed_samples": 3671, | |
| "total_samples": 3671, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.7583, | |
| "_scorer/std": 0.0055, | |
| "_scorer/criteria_met_rate": 0.7605, | |
| "healthbench_score/mean": 0.7583, | |
| "healthbench_score_length_adjusted/mean": 0.7522 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-4b-it", | |
| "openai/gpt-4.1" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-4b-it": 2959598, | |
| "openai/gpt-4.1": 12979748 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-hard-medgemma-v2", | |
| "file": "2026-08-05T07-54-55-00-00_healthbench-hard_MANxy36rpJhotpAvdwQiFk.eval", | |
| "task": "healthbench_hard", | |
| "task_version": 2, | |
| "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", | |
| "model": "vllm/google/medgemma-4b-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "judge_model": "openai/gpt-4.1", | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-08-05T07:54:55+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.252" | |
| }, | |
| "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0, | |
| "_scorer/std": 0.0, | |
| "_scorer/criteria_met_rate": 0.3527, | |
| "healthbench_score/mean": -0.0347, | |
| "healthbench_score_length_adjusted/mean": -0.1256 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-4b-it", | |
| "openai/gpt-4.1" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-4b-it": 887308, | |
| "openai/gpt-4.1": 19808558 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-hard-medgemma-v2", | |
| "file": "2026-08-05T08-07-41-00-00_healthbench-hard_D9KnFzsoQRE57vVsejUQnW.eval", | |
| "task": "healthbench_hard", | |
| "task_version": 2, | |
| "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "judge_model": "openai/gpt-4.1", | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-08-05T08:07:41+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.252" | |
| }, | |
| "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.1415, | |
| "_scorer/std": 0.0099, | |
| "_scorer/criteria_met_rate": 0.4468, | |
| "healthbench_score/mean": 0.1415, | |
| "healthbench_score_length_adjusted/mean": -0.023 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-27b-text-it", | |
| "openai/gpt-4.1" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-27b-text-it": 1130260, | |
| "openai/gpt-4.1": 22994024 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-professional-deepseek-v4-pro", | |
| "file": "2026-08-06T02-35-30-00-00_healthbench-professional_bbeHCEMNSj6Q362BQHa7mS.eval", | |
| "task": "healthbench_professional", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "local_path": null, | |
| "hf": true, | |
| "use_case": null, | |
| "usage_type": null, | |
| "difficulty": null, | |
| "specialty": null, | |
| "physician_baseline": false, | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "judge_reasoning_effort": "low", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": true, | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0147, | |
| "cache": true | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "retry_on_error": 3, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-08-06T02:35:30+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.252" | |
| }, | |
| "revision": null, | |
| "completed_samples": 525, | |
| "total_samples": 525, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.3099, | |
| "_scorer/std": 0.0279, | |
| "_scorer/criteria_met_rate": 0.5921, | |
| "_scorer/use_case_consult_score": 0.4605, | |
| "_scorer/use_case_consult_std": 0.0305, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.666, | |
| "_scorer/use_case_research_std": 0.0324, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.3543, | |
| "_scorer/use_case_writing_std": 0.036, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.6265, | |
| "_scorer/type_good_faith_std": 0.0226, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.2494, | |
| "_scorer/type_red_teaming_std": 0.0298, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.7023, | |
| "_scorer/difficulty_typical_std": 0.0225, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.2867, | |
| "_scorer/difficulty_difficult_std": 0.0256, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.4631, | |
| "_scorer/specialty_ent_std": 0.1645, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.338, | |
| "_scorer/specialty_gi_std": 0.1003, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.3996, | |
| "_scorer/specialty_id_std": 0.1037, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.58, | |
| "_scorer/specialty_anesthesia_std": 0.0903, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.4885, | |
| "_scorer/specialty_cards_std": 0.0681, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.4581, | |
| "_scorer/specialty_derm_std": 0.0919, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.4193, | |
| "_scorer/specialty_emergency_std": 0.1189, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5738, | |
| "_scorer/specialty_endo_std": 0.0944, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.875, | |
| "_scorer/specialty_genetics_std": 0.1125, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.3914, | |
| "_scorer/specialty_heme/onc_std": 0.085, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.3012, | |
| "_scorer/specialty_medicine_std": 0.1004, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.6434, | |
| "_scorer/specialty_nephro_std": 0.0823, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.6737, | |
| "_scorer/specialty_neuro_std": 0.0591, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.5385, | |
| "_scorer/specialty_neurosurg_std": 0.1299, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.4391, | |
| "_scorer/specialty_obgyn_std": 0.0627, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.5012, | |
| "_scorer/specialty_ophtho_std": 0.1049, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.4772, | |
| "_scorer/specialty_ortho_std": 0.0912, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.4223, | |
| "_scorer/specialty_other_std": 0.1018, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.4566, | |
| "_scorer/specialty_peds_std": 0.0911, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.4501, | |
| "_scorer/specialty_primary_std": 0.16, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.594, | |
| "_scorer/specialty_psychiatry_std": 0.0852, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.2632, | |
| "_scorer/specialty_pulm_std": 0.1066, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.4375, | |
| "_scorer/specialty_rads_std": 0.1592, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.375, | |
| "_scorer/specialty_rheum_std": 0.1675, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.6471, | |
| "_scorer/specialty_surgery_std": 0.1192, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.5619, | |
| "_scorer/specialty_uro_std": 0.1296, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7023, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0222, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3778, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0544, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2494, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0297, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.3781, | |
| "healthbench_professional_score_length_adjusted/mean": 0.3099 | |
| }, | |
| "model_usage_models": [ | |
| "openrouter/deepseek/deepseek-v4-pro", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "openrouter/deepseek/deepseek-v4-pro": 1258221, | |
| "openai/gpt-5.4": 2810449 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-consensus", | |
| "file": "2026-07-16T20-09-58-00-00_healthbench-consensus_87a5hkWgWY5rrHyoygzDVx.eval", | |
| "task": "inspect_evals/healthbench_consensus", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai-api/plamo/plamo-3.0-prime", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.002 | |
| }, | |
| "dataset_samples": 3671, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 183, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T20:09:58+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 3671, | |
| "total_samples": 3671, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.7525, | |
| "_scorer/std": 0.0052, | |
| "_scorer/criteria_met_rate": 0.7369, | |
| "healthbench_score/mean": 0.7525, | |
| "healthbench_score_length_adjusted/mean": 0.7478 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4o-mini": 12531224 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-consensus", | |
| "file": "2026-07-16T20-20-37-00-00_healthbench-consensus_gmfrK88PFGpuvRWqnk7uc5.eval", | |
| "task": "inspect_evals/healthbench_consensus", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.002 | |
| }, | |
| "dataset_samples": 3671, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 183, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T20:20:37+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 3671, | |
| "total_samples": 3671, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.8208, | |
| "_scorer/std": 0.0046, | |
| "_scorer/criteria_met_rate": 0.8079, | |
| "healthbench_score/mean": 0.8208, | |
| "healthbench_score_length_adjusted/mean": 0.8202 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4o-mini": 10709027 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-consensus", | |
| "file": "2026-07-16T20-44-07-00-00_healthbench-consensus_EbuNesZCwbaRVAqRedBwjP.eval", | |
| "task": "inspect_evals/healthbench_consensus", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.002 | |
| }, | |
| "dataset_samples": 3671, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 183, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T20:44:07+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 3671, | |
| "total_samples": 3671, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.791, | |
| "_scorer/std": 0.0049, | |
| "_scorer/criteria_met_rate": 0.7765, | |
| "healthbench_score/mean": 0.791, | |
| "healthbench_score_length_adjusted/mean": 0.7846 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4o-mini": 13025085 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-consensus", | |
| "file": "2026-07-16T21-11-43-00-00_healthbench-consensus_k2nEohAktRtjKfZ5BNGJSG.eval", | |
| "task": "inspect_evals/healthbench_consensus", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.002 | |
| }, | |
| "dataset_samples": 3671, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 183, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T21:11:43+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 3671, | |
| "total_samples": 3671, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.8017, | |
| "_scorer/std": 0.0049, | |
| "_scorer/criteria_met_rate": 0.7954, | |
| "healthbench_score/mean": 0.8017, | |
| "healthbench_score_length_adjusted/mean": 0.8023 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4o-mini": 10263280 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-consensus", | |
| "file": "2026-07-24T09-07-55-00-00_healthbench-consensus_QX6zAfF9oDZDAX4p6k6yb9.eval", | |
| "task": "inspect_evals/healthbench_consensus", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "vllm/google/medgemma-4b-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.002 | |
| }, | |
| "dataset_samples": 3671, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 183, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T09:07:55+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249", | |
| "inspect_evals": "0.16.0" | |
| }, | |
| "revision": null, | |
| "completed_samples": 3671, | |
| "total_samples": 3671, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.7137, | |
| "_scorer/std": 0.006, | |
| "_scorer/criteria_met_rate": 0.7011, | |
| "healthbench_score/mean": 0.7137, | |
| "healthbench_score_length_adjusted/mean": 0.7077 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-4b-it", | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-4b-it": 3440824, | |
| "openai/gpt-4o-mini": 14762331 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-consensus", | |
| "file": "2026-07-24T11-39-48-00-00_healthbench-consensus_7b4ZMKGYfFkgFNSJXJs53z.eval", | |
| "task": "inspect_evals/healthbench_consensus", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.002 | |
| }, | |
| "dataset_samples": 3671, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 183, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T11:39:48+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249", | |
| "inspect_evals": "0.16.0" | |
| }, | |
| "revision": null, | |
| "completed_samples": 3671, | |
| "total_samples": 3671, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.7758, | |
| "_scorer/std": 0.0049, | |
| "_scorer/criteria_met_rate": 0.7616, | |
| "healthbench_score/mean": 0.7758, | |
| "healthbench_score_length_adjusted/mean": 0.7661 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-27b-text-it", | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-27b-text-it": 4303064, | |
| "openai/gpt-4o-mini": 16607219 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-hard", | |
| "file": "2026-07-16T07-28-45-00-00_healthbench-hard_dxTyk6Wmosx3kJAp4iULs9.eval", | |
| "task": "inspect_evals/healthbench_hard", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T07:28:45+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2727, | |
| "_scorer/std": 0.0099, | |
| "_scorer/criteria_met_rate": 0.3811, | |
| "healthbench_score/mean": 0.2727, | |
| "healthbench_score_length_adjusted/mean": 0.2603 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4o-mini": 17512569 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-hard", | |
| "file": "2026-07-16T18-52-11-00-00_healthbench-hard_Z9QDknMd4SukiAP7Q4FSPz.eval", | |
| "task": "inspect_evals/healthbench_hard", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T18:52:11+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2661, | |
| "_scorer/std": 0.0095, | |
| "_scorer/criteria_met_rate": 0.3611, | |
| "healthbench_score/mean": 0.2661, | |
| "healthbench_score_length_adjusted/mean": 0.278 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4o-mini": 16665282 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-hard", | |
| "file": "2026-07-16T19-07-31-00-00_healthbench-hard_SABwaQjc3Xon5iDMpWFQV3.eval", | |
| "task": "inspect_evals/healthbench_hard", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T19:07:31+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2484, | |
| "_scorer/std": 0.0096, | |
| "_scorer/criteria_met_rate": 0.354, | |
| "healthbench_score/mean": 0.2484, | |
| "healthbench_score_length_adjusted/mean": 0.138 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4o-mini": 21037790 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-hard", | |
| "file": "2026-07-16T19-23-40-00-00_healthbench-hard_mEs7dJHDapQ4JVqHQtfTJc.eval", | |
| "task": "inspect_evals/healthbench_hard", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai-api/plamo/plamo-3.0-prime", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T19:23:40+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.1743, | |
| "_scorer/std": 0.0084, | |
| "_scorer/criteria_met_rate": 0.2732, | |
| "healthbench_score/mean": 0.1743, | |
| "healthbench_score_length_adjusted/mean": 0.0961 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4o-mini": 20067380 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-hard", | |
| "file": "2026-07-24T09-18-15-00-00_healthbench-hard_WkGg4UV64qpccys2fbViHY.eval", | |
| "task": "inspect_evals/healthbench_hard", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "vllm/google/medgemma-4b-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T09:18:15+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249", | |
| "inspect_evals": "0.16.0" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.106, | |
| "_scorer/std": 0.0077, | |
| "_scorer/criteria_met_rate": 0.2222, | |
| "healthbench_score/mean": 0.106, | |
| "healthbench_score_length_adjusted/mean": 0.0126 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-4b-it", | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-4b-it": 402479, | |
| "openai/gpt-4o-mini": 17583141 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-hard", | |
| "file": "2026-07-24T14-36-42-00-00_healthbench-hard_BU7wDz79JMGuziKwJrRrpN.eval", | |
| "task": "inspect_evals/healthbench_hard", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T14:36:42+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249", | |
| "inspect_evals": "0.16.0" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2109, | |
| "_scorer/std": 0.0083, | |
| "_scorer/criteria_met_rate": 0.3143, | |
| "healthbench_score/mean": 0.2109, | |
| "healthbench_score_length_adjusted/mean": 0.0483 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4o-mini", | |
| "vllm/google/medgemma-27b-text-it" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4o-mini": 20040174, | |
| "vllm/google/medgemma-27b-text-it": 24135 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-non-professional", | |
| "file": "2026-07-15T06-51-58-00-00_healthbench_WJDhL9LgZJKMGgYizSLYpL.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4.1", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0299 | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-15T06:51:58+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 5000, | |
| "total_samples": 5000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.5687, | |
| "_scorer/std": 0.0048, | |
| "_scorer/criteria_met_rate": 0.5365, | |
| "healthbench_score/mean": 0.5687, | |
| "healthbench_score_length_adjusted/mean": 0.5583 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-non-professional", | |
| "file": "2026-07-15T07-01-24-00-00_healthbench_Hah9CRDdSrTz7K5EFgRWF8.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4.1", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0299 | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-15T07:01:24+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 5000, | |
| "total_samples": 5000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.5344, | |
| "_scorer/std": 0.0048, | |
| "_scorer/criteria_met_rate": 0.5323, | |
| "healthbench_score/mean": 0.5344, | |
| "healthbench_score_length_adjusted/mean": 0.5431 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-non-professional", | |
| "file": "2026-07-16T04-29-43-00-00_healthbench_jBnP8VSE7rSvrMpzRStQFK.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4.1", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0299 | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T04:29:43+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 5000, | |
| "total_samples": 5000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.5137, | |
| "_scorer/std": 0.0051, | |
| "_scorer/criteria_met_rate": 0.542, | |
| "healthbench_score/mean": 0.5137, | |
| "healthbench_score_length_adjusted/mean": 0.4173 | |
| }, | |
| "model_usage_models": [ | |
| "openrouter/deepseek/deepseek-v4-pro", | |
| "openai/gpt-4.1" | |
| ], | |
| "model_usage_total": { | |
| "openrouter/deepseek/deepseek-v4-pro": 7415173, | |
| "openai/gpt-4.1": 105583667 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-non-professional", | |
| "file": "2026-07-16T05-23-21-00-00_healthbench_iexipStaaxm8E5ni3Gg96Z.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai-api/plamo/plamo-3.0-prime", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4.1", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0299 | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-16T05:23:21+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 5000, | |
| "total_samples": 5000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.3942, | |
| "_scorer/std": 0.0054, | |
| "_scorer/criteria_met_rate": 0.4698, | |
| "healthbench_score/mean": 0.3942, | |
| "healthbench_score_length_adjusted/mean": 0.3243 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4.1", | |
| "openai-api/plamo/plamo-3.0-prime" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4.1": 49156082, | |
| "openai-api/plamo/plamo-3.0-prime": 2449482 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-non-professional", | |
| "file": "2026-07-24T09-32-53-00-00_healthbench_6Co8iMuXsKxGNrMeAkRbdp.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "vllm/google/medgemma-4b-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4.1", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0299 | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T09:32:53+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249", | |
| "inspect_evals": "0.16.0" | |
| }, | |
| "revision": null, | |
| "completed_samples": 5000, | |
| "total_samples": 5000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2697, | |
| "_scorer/std": 0.0057, | |
| "_scorer/criteria_met_rate": 0.4107, | |
| "healthbench_score/mean": 0.2697, | |
| "healthbench_score_length_adjusted/mean": 0.1832 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-4b-it", | |
| "openai/gpt-4.1" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-4b-it": 877814, | |
| "openai/gpt-4.1": 100167550 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-non-professional", | |
| "file": "2026-07-24T12-08-49-00-00_healthbench_cuBWJL4bbU32vR5cxHsjEC.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4.1", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0299 | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T12:08:49+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249", | |
| "inspect_evals": "0.16.0" | |
| }, | |
| "revision": null, | |
| "completed_samples": 5000, | |
| "total_samples": 5000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.472, | |
| "_scorer/std": 0.0054, | |
| "_scorer/criteria_met_rate": 0.5262, | |
| "healthbench_score/mean": 0.472, | |
| "healthbench_score_length_adjusted/mean": 0.3321 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-4.1", | |
| "vllm/google/medgemma-27b-text-it" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-4.1": 113164489, | |
| "vllm/google/medgemma-27b-text-it": 1556281 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-original", | |
| "file": "2026-07-24T15-57-49-00-00_healthbench-professional_7PZFsBJYTZRCdXZkJPfki7.eval", | |
| "task": "healthbench_professional", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "local_path": null, | |
| "hf": true, | |
| "use_case": null, | |
| "usage_type": null, | |
| "difficulty": null, | |
| "specialty": null, | |
| "physician_baseline": false, | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "judge_reasoning_effort": "low", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": true, | |
| "length_adjustment_center": 2000.0, | |
| "length_adjustment_penalty_per_500_chars": 0.0147, | |
| "cache": true | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T15:57:49+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4781, | |
| "_scorer/std": 0.0226, | |
| "_scorer/criteria_met_rate": 0.635, | |
| "_scorer/use_case_consult_score": 0.6058, | |
| "_scorer/use_case_consult_std": 0.0292, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.7055, | |
| "_scorer/use_case_research_std": 0.0308, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.5245, | |
| "_scorer/use_case_writing_std": 0.0379, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.6996, | |
| "_scorer/type_good_faith_std": 0.0213, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.458, | |
| "_scorer/type_red_teaming_std": 0.0339, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.757, | |
| "_scorer/difficulty_typical_std": 0.0215, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.4735, | |
| "_scorer/difficulty_difficult_std": 0.0284, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.7583, | |
| "_scorer/specialty_ent_std": 0.1338, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.4199, | |
| "_scorer/specialty_gi_std": 0.093, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.513, | |
| "_scorer/specialty_id_std": 0.0837, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.691, | |
| "_scorer/specialty_anesthesia_std": 0.0873, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.5371, | |
| "_scorer/specialty_cards_std": 0.0706, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.5994, | |
| "_scorer/specialty_derm_std": 0.0884, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.6375, | |
| "_scorer/specialty_emergency_std": 0.101, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.7583, | |
| "_scorer/specialty_endo_std": 0.0876, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.75, | |
| "_scorer/specialty_genetics_std": 0.2205, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.5129, | |
| "_scorer/specialty_heme/onc_std": 0.0819, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.4342, | |
| "_scorer/specialty_medicine_std": 0.113, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.8512, | |
| "_scorer/specialty_nephro_std": 0.0663, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.7139, | |
| "_scorer/specialty_neuro_std": 0.0533, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.4606, | |
| "_scorer/specialty_neurosurg_std": 0.1454, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.6554, | |
| "_scorer/specialty_obgyn_std": 0.0635, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.6142, | |
| "_scorer/specialty_ophtho_std": 0.1078, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.4436, | |
| "_scorer/specialty_ortho_std": 0.0902, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.4854, | |
| "_scorer/specialty_other_std": 0.1011, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.3333, | |
| "_scorer/specialty_path_std": 0.2852, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.536, | |
| "_scorer/specialty_peds_std": 0.0836, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.7815, | |
| "_scorer/specialty_primary_std": 0.1417, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.678, | |
| "_scorer/specialty_psychiatry_std": 0.0807, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.7483, | |
| "_scorer/specialty_pulm_std": 0.0939, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.6875, | |
| "_scorer/specialty_rads_std": 0.1503, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.5, | |
| "_scorer/specialty_rheum_std": 0.1817, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.7353, | |
| "_scorer/specialty_surgery_std": 0.1033, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.6538, | |
| "_scorer/specialty_uro_std": 0.1219, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.757, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0211, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.5114, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0527, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.458, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0352, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.5295, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4781 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-5.5", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-5.5": 8994839, | |
| "openai/gpt-5.4": 21393751 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-original", | |
| "file": "2026-07-24T16-55-39-00-00_healthbench-professional_boiooCVNG7ZkBVsnEpKLNT.eval", | |
| "task": "healthbench_professional", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "local_path": null, | |
| "hf": true, | |
| "use_case": null, | |
| "usage_type": null, | |
| "difficulty": null, | |
| "specialty": null, | |
| "physician_baseline": false, | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "judge_reasoning_effort": "low", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": true, | |
| "length_adjustment_center": 2000.0, | |
| "length_adjustment_penalty_per_500_chars": 0.0147, | |
| "cache": true | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T16:55:39+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4803, | |
| "_scorer/std": 0.0235, | |
| "_scorer/criteria_met_rate": 0.6238, | |
| "_scorer/use_case_consult_score": 0.6051, | |
| "_scorer/use_case_consult_std": 0.028, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.6739, | |
| "_scorer/use_case_research_std": 0.0339, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.5091, | |
| "_scorer/use_case_writing_std": 0.037, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.6785, | |
| "_scorer/type_good_faith_std": 0.0212, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.4583, | |
| "_scorer/type_red_teaming_std": 0.0335, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.7291, | |
| "_scorer/difficulty_typical_std": 0.0219, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.474, | |
| "_scorer/difficulty_difficult_std": 0.03, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.6821, | |
| "_scorer/specialty_ent_std": 0.1682, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.4119, | |
| "_scorer/specialty_gi_std": 0.0897, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.6207, | |
| "_scorer/specialty_id_std": 0.0845, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 0.5, | |
| "_scorer/specialty_allergy_std": 0.3595, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.7658, | |
| "_scorer/specialty_anesthesia_std": 0.0746, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.5808, | |
| "_scorer/specialty_cards_std": 0.0634, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.6161, | |
| "_scorer/specialty_derm_std": 0.0883, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.5294, | |
| "_scorer/specialty_emergency_std": 0.1183, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.6903, | |
| "_scorer/specialty_endo_std": 0.0944, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.4205, | |
| "_scorer/specialty_genetics_std": 0.2161, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.4822, | |
| "_scorer/specialty_heme/onc_std": 0.0857, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.5676, | |
| "_scorer/specialty_medicine_std": 0.106, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.7864, | |
| "_scorer/specialty_nephro_std": 0.0689, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.7189, | |
| "_scorer/specialty_neuro_std": 0.0531, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.5109, | |
| "_scorer/specialty_neurosurg_std": 0.1316, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.5429, | |
| "_scorer/specialty_obgyn_std": 0.0685, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.5586, | |
| "_scorer/specialty_ophtho_std": 0.1146, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.5932, | |
| "_scorer/specialty_ortho_std": 0.0957, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.5797, | |
| "_scorer/specialty_other_std": 0.0941, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.1905, | |
| "_scorer/specialty_path_std": 0.1571, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.4692, | |
| "_scorer/specialty_peds_std": 0.0858, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.6387, | |
| "_scorer/specialty_primary_std": 0.1686, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.6688, | |
| "_scorer/specialty_psychiatry_std": 0.0784, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.5967, | |
| "_scorer/specialty_pulm_std": 0.1183, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.7463, | |
| "_scorer/specialty_rads_std": 0.1261, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.5, | |
| "_scorer/specialty_rheum_std": 0.17, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.6765, | |
| "_scorer/specialty_surgery_std": 0.1096, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.511, | |
| "_scorer/specialty_uro_std": 0.1318, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7291, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0206, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.5125, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0534, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.4583, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.034, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.5083, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4803 | |
| }, | |
| "model_usage_models": [ | |
| "anthropic/claude-opus-4-7", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "anthropic/claude-opus-4-7": 9075443, | |
| "openai/gpt-5.4": 19817644 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-original", | |
| "file": "2026-07-24T17-30-37-00-00_healthbench-professional_c9RnXrEaoi4T33bvwXEA5h.eval", | |
| "task": "healthbench_professional", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai-api/plamo/plamo-3.0-prime", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "local_path": null, | |
| "hf": true, | |
| "use_case": null, | |
| "usage_type": null, | |
| "difficulty": null, | |
| "specialty": null, | |
| "physician_baseline": false, | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "judge_reasoning_effort": "low", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": true, | |
| "length_adjustment_center": 2000.0, | |
| "length_adjustment_penalty_per_500_chars": 0.0147, | |
| "cache": true | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T17:30:37+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.1374, | |
| "_scorer/std": 0.0236, | |
| "_scorer/criteria_met_rate": 0.4202, | |
| "_scorer/use_case_consult_score": 0.3313, | |
| "_scorer/use_case_consult_std": 0.026, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.4886, | |
| "_scorer/use_case_research_std": 0.0334, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.1977, | |
| "_scorer/use_case_writing_std": 0.0285, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.4366, | |
| "_scorer/type_good_faith_std": 0.0231, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.1688, | |
| "_scorer/type_red_teaming_std": 0.0255, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.4965, | |
| "_scorer/difficulty_typical_std": 0.0243, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.1894, | |
| "_scorer/difficulty_difficult_std": 0.0225, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.4857, | |
| "_scorer/specialty_ent_std": 0.1744, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.1772, | |
| "_scorer/specialty_gi_std": 0.0766, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.2871, | |
| "_scorer/specialty_id_std": 0.0833, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.3928, | |
| "_scorer/specialty_anesthesia_std": 0.0903, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.4052, | |
| "_scorer/specialty_cards_std": 0.0608, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.2958, | |
| "_scorer/specialty_derm_std": 0.0803, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.4044, | |
| "_scorer/specialty_emergency_std": 0.1205, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5108, | |
| "_scorer/specialty_endo_std": 0.0907, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.75, | |
| "_scorer/specialty_genetics_std": 0.2168, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.2388, | |
| "_scorer/specialty_heme/onc_std": 0.0755, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.2962, | |
| "_scorer/specialty_medicine_std": 0.1038, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.4423, | |
| "_scorer/specialty_nephro_std": 0.0844, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.4003, | |
| "_scorer/specialty_neuro_std": 0.062, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.4342, | |
| "_scorer/specialty_neurosurg_std": 0.1386, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.3389, | |
| "_scorer/specialty_obgyn_std": 0.0689, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.3904, | |
| "_scorer/specialty_ophtho_std": 0.1091, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.3319, | |
| "_scorer/specialty_ortho_std": 0.0895, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.2228, | |
| "_scorer/specialty_other_std": 0.0781, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.3333, | |
| "_scorer/specialty_path_std": 0.272, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.2704, | |
| "_scorer/specialty_peds_std": 0.0761, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.4958, | |
| "_scorer/specialty_primary_std": 0.1766, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.2619, | |
| "_scorer/specialty_psychiatry_std": 0.0695, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.2189, | |
| "_scorer/specialty_pulm_std": 0.0956, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.1875, | |
| "_scorer/specialty_rads_std": 0.1278, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.3601, | |
| "_scorer/specialty_rheum_std": 0.1394, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.3216, | |
| "_scorer/specialty_surgery_std": 0.1044, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.3156, | |
| "_scorer/specialty_uro_std": 0.0998, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.4965, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0249, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.2399, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.046, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.1688, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0264, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.2078, | |
| "healthbench_professional_score_length_adjusted/mean": 0.1374 | |
| }, | |
| "model_usage_models": [ | |
| "openai-api/plamo/plamo-3.0-prime", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "openai-api/plamo/plamo-3.0-prime": 8130263, | |
| "openai/gpt-5.4": 23382174 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-original", | |
| "file": "2026-07-24T22-44-03-00-00_healthbench-professional_6AYXiEDgGuzNtAcyWPDGrR.eval", | |
| "task": "healthbench_professional", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "local_path": null, | |
| "hf": true, | |
| "use_case": null, | |
| "usage_type": null, | |
| "difficulty": null, | |
| "specialty": null, | |
| "physician_baseline": false, | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "judge_reasoning_effort": "low", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": true, | |
| "length_adjustment_center": 2000.0, | |
| "length_adjustment_penalty_per_500_chars": 0.0147, | |
| "cache": true | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T22:44:03+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2003, | |
| "_scorer/std": 0.0242, | |
| "_scorer/criteria_met_rate": 0.4928, | |
| "_scorer/use_case_consult_score": 0.3894, | |
| "_scorer/use_case_consult_std": 0.0283, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.5213, | |
| "_scorer/use_case_research_std": 0.0327, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.3879, | |
| "_scorer/use_case_writing_std": 0.0375, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.5249, | |
| "_scorer/type_good_faith_std": 0.0231, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.2529, | |
| "_scorer/type_red_teaming_std": 0.0283, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.5851, | |
| "_scorer/difficulty_typical_std": 0.0235, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.2745, | |
| "_scorer/difficulty_difficult_std": 0.0263, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.6155, | |
| "_scorer/specialty_ent_std": 0.1543, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.1627, | |
| "_scorer/specialty_gi_std": 0.0714, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.426, | |
| "_scorer/specialty_id_std": 0.0819, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.4063, | |
| "_scorer/specialty_anesthesia_std": 0.0882, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.4054, | |
| "_scorer/specialty_cards_std": 0.0595, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.471, | |
| "_scorer/specialty_derm_std": 0.0919, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.5021, | |
| "_scorer/specialty_emergency_std": 0.1173, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5296, | |
| "_scorer/specialty_endo_std": 0.0885, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.75, | |
| "_scorer/specialty_genetics_std": 0.2164, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.3479, | |
| "_scorer/specialty_heme/onc_std": 0.0773, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.3804, | |
| "_scorer/specialty_medicine_std": 0.115, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.5785, | |
| "_scorer/specialty_nephro_std": 0.0882, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.4748, | |
| "_scorer/specialty_neuro_std": 0.0612, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.5073, | |
| "_scorer/specialty_neurosurg_std": 0.1282, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.403, | |
| "_scorer/specialty_obgyn_std": 0.0673, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.4915, | |
| "_scorer/specialty_ophtho_std": 0.1119, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.481, | |
| "_scorer/specialty_ortho_std": 0.0913, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.388, | |
| "_scorer/specialty_other_std": 0.1, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.3969, | |
| "_scorer/specialty_peds_std": 0.0854, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.563, | |
| "_scorer/specialty_primary_std": 0.1527, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.4225, | |
| "_scorer/specialty_psychiatry_std": 0.0769, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.3023, | |
| "_scorer/specialty_pulm_std": 0.105, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.375, | |
| "_scorer/specialty_rads_std": 0.1699, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.3542, | |
| "_scorer/specialty_rheum_std": 0.1343, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.4098, | |
| "_scorer/specialty_surgery_std": 0.1092, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.4106, | |
| "_scorer/specialty_uro_std": 0.1189, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.5851, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0242, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3273, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0509, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2529, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0283, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.3125, | |
| "healthbench_professional_score_length_adjusted/mean": 0.2003 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-5.4", | |
| "vllm/google/medgemma-27b-text-it" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-5.4": 25383913, | |
| "vllm/google/medgemma-27b-text-it": 8383935 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-original", | |
| "file": "2026-07-25T00-26-29-00-00_healthbench-professional_GEViyNR2PM2B2oUV94DzoK.eval", | |
| "task": "healthbench_professional", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "vllm/google/medgemma-4b-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "local_path": null, | |
| "hf": true, | |
| "use_case": null, | |
| "usage_type": null, | |
| "difficulty": null, | |
| "specialty": null, | |
| "physician_baseline": false, | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "judge_reasoning_effort": "low", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": true, | |
| "length_adjustment_center": 2000.0, | |
| "length_adjustment_penalty_per_500_chars": 0.0147, | |
| "cache": true | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-25T00:26:29+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0904, | |
| "_scorer/std": 0.0218, | |
| "_scorer/criteria_met_rate": 0.353, | |
| "_scorer/use_case_consult_score": 0.2765, | |
| "_scorer/use_case_consult_std": 0.0255, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.378, | |
| "_scorer/use_case_research_std": 0.0331, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.2347, | |
| "_scorer/use_case_writing_std": 0.0317, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.3664, | |
| "_scorer/type_good_faith_std": 0.0218, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.1664, | |
| "_scorer/type_red_teaming_std": 0.0262, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.4352, | |
| "_scorer/difficulty_typical_std": 0.0251, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.1589, | |
| "_scorer/difficulty_difficult_std": 0.0205, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.4381, | |
| "_scorer/specialty_ent_std": 0.1637, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.1968, | |
| "_scorer/specialty_gi_std": 0.076, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.2115, | |
| "_scorer/specialty_id_std": 0.0761, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.2753, | |
| "_scorer/specialty_anesthesia_std": 0.0799, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.326, | |
| "_scorer/specialty_cards_std": 0.061, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.3973, | |
| "_scorer/specialty_derm_std": 0.085, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.4982, | |
| "_scorer/specialty_emergency_std": 0.117, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.4903, | |
| "_scorer/specialty_endo_std": 0.0787, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.6705, | |
| "_scorer/specialty_genetics_std": 0.1998, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.1647, | |
| "_scorer/specialty_heme/onc_std": 0.0648, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.1889, | |
| "_scorer/specialty_medicine_std": 0.0717, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.3844, | |
| "_scorer/specialty_nephro_std": 0.0833, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.3356, | |
| "_scorer/specialty_neuro_std": 0.0542, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.3848, | |
| "_scorer/specialty_neurosurg_std": 0.1239, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.2405, | |
| "_scorer/specialty_obgyn_std": 0.0597, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.3773, | |
| "_scorer/specialty_ophtho_std": 0.1088, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.2661, | |
| "_scorer/specialty_ortho_std": 0.0772, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.3561, | |
| "_scorer/specialty_other_std": 0.0932, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.2501, | |
| "_scorer/specialty_peds_std": 0.0711, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.3745, | |
| "_scorer/specialty_primary_std": 0.1358, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.1877, | |
| "_scorer/specialty_psychiatry_std": 0.0602, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.148, | |
| "_scorer/specialty_pulm_std": 0.0831, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.0625, | |
| "_scorer/specialty_rads_std": 0.0575, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.2917, | |
| "_scorer/specialty_rheum_std": 0.1439, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.3235, | |
| "_scorer/specialty_surgery_std": 0.1087, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.214, | |
| "_scorer/specialty_uro_std": 0.086, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.4352, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0245, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.1407, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0342, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.1664, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0245, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.1651, | |
| "healthbench_professional_score_length_adjusted/mean": 0.0904 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-4b-it", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-4b-it": 7090153, | |
| "openai/gpt-5.4": 22550200 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-original", | |
| "file": "2026-07-25T18-24-55-00-00_healthbench-professional_HaKmMaK4Q6JCLuX2RLCPnc.eval", | |
| "task": "healthbench_professional", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "local_path": null, | |
| "hf": true, | |
| "use_case": null, | |
| "usage_type": null, | |
| "difficulty": null, | |
| "specialty": null, | |
| "physician_baseline": false, | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "judge_reasoning_effort": "low", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": true, | |
| "length_adjustment_center": 2000.0, | |
| "length_adjustment_penalty_per_500_chars": 0.0147, | |
| "cache": true | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": 0.1, | |
| "continue_on_fail": false, | |
| "retry_on_error": 5, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-25T18:24:55+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 525, | |
| "total_samples": 525, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2742, | |
| "_scorer/std": 0.0264, | |
| "_scorer/criteria_met_rate": 0.5921, | |
| "_scorer/use_case_consult_score": 0.4169, | |
| "_scorer/use_case_consult_std": 0.0279, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.6383, | |
| "_scorer/use_case_research_std": 0.0321, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.3413, | |
| "_scorer/use_case_writing_std": 0.0367, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.5995, | |
| "_scorer/type_good_faith_std": 0.0219, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.2117, | |
| "_scorer/type_red_teaming_std": 0.0289, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.6716, | |
| "_scorer/difficulty_typical_std": 0.0233, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.2555, | |
| "_scorer/difficulty_difficult_std": 0.0245, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.6821, | |
| "_scorer/specialty_ent_std": 0.1635, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.2094, | |
| "_scorer/specialty_gi_std": 0.0862, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.403, | |
| "_scorer/specialty_id_std": 0.0984, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.6112, | |
| "_scorer/specialty_anesthesia_std": 0.0934, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.4409, | |
| "_scorer/specialty_cards_std": 0.066, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.4885, | |
| "_scorer/specialty_derm_std": 0.0898, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.3237, | |
| "_scorer/specialty_emergency_std": 0.1081, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5544, | |
| "_scorer/specialty_endo_std": 0.0905, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.5455, | |
| "_scorer/specialty_genetics_std": 0.18, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.3489, | |
| "_scorer/specialty_heme/onc_std": 0.0825, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.2598, | |
| "_scorer/specialty_medicine_std": 0.0969, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.5412, | |
| "_scorer/specialty_nephro_std": 0.0855, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.5966, | |
| "_scorer/specialty_neuro_std": 0.0597, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.4961, | |
| "_scorer/specialty_neurosurg_std": 0.1267, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.4996, | |
| "_scorer/specialty_obgyn_std": 0.0638, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.5038, | |
| "_scorer/specialty_ophtho_std": 0.1129, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.4117, | |
| "_scorer/specialty_ortho_std": 0.0888, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.4047, | |
| "_scorer/specialty_other_std": 0.0994, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.4606, | |
| "_scorer/specialty_peds_std": 0.0945, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.4501, | |
| "_scorer/specialty_primary_std": 0.159, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.5628, | |
| "_scorer/specialty_psychiatry_std": 0.0791, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.3439, | |
| "_scorer/specialty_pulm_std": 0.1118, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.3708, | |
| "_scorer/specialty_rads_std": 0.1446, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.4524, | |
| "_scorer/specialty_rheum_std": 0.164, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.5294, | |
| "_scorer/specialty_surgery_std": 0.1219, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.3721, | |
| "_scorer/specialty_uro_std": 0.1228, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.6716, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0218, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3629, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.049, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2117, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0269, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.343, | |
| "healthbench_professional_score_length_adjusted/mean": 0.2742 | |
| }, | |
| "model_usage_models": [ | |
| "openrouter/deepseek/deepseek-v4-pro", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "openrouter/deepseek/deepseek-v4-pro": 5338, | |
| "openai/gpt-5.4": 8795 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T16-36-59-00-00_healthbench-professional-consult_D8Z93ZWFtkR4CDbDBLZvBT.eval", | |
| "task": "healthbench_professional_consult", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 236, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 94, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T16:36:59+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1888, | |
| "total_samples": 1888, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4864, | |
| "_scorer/std": 0.0381, | |
| "_scorer/criteria_met_rate": 0.6351, | |
| "_scorer/use_case_consult_score": 0.6058, | |
| "_scorer/use_case_consult_std": 0.0292, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/type_good_faith_score": 0.6676, | |
| "_scorer/type_good_faith_std": 0.0355, | |
| "_scorer/type_good_faith_n_samples": 143.0, | |
| "_scorer/type_red_teaming_score": 0.5109, | |
| "_scorer/type_red_teaming_std": 0.0504, | |
| "_scorer/type_red_teaming_n_samples": 93.0, | |
| "_scorer/difficulty_typical_score": 0.7806, | |
| "_scorer/difficulty_typical_std": 0.0353, | |
| "_scorer/difficulty_typical_n_samples": 84.0, | |
| "_scorer/difficulty_difficult_score": 0.5093, | |
| "_scorer/difficulty_difficult_std": 0.0374, | |
| "_scorer/difficulty_difficult_n_samples": 152.0, | |
| "_scorer/specialty_ent_score": 0.8271, | |
| "_scorer/specialty_ent_std": 0.0948, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.4375, | |
| "_scorer/specialty_gi_std": 0.1593, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.6133, | |
| "_scorer/specialty_id_std": 0.1398, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.7406, | |
| "_scorer/specialty_anesthesia_std": 0.1203, | |
| "_scorer/specialty_anesthesia_n_samples": 10.0, | |
| "_scorer/specialty_cards_score": 0.3748, | |
| "_scorer/specialty_cards_std": 0.0921, | |
| "_scorer/specialty_cards_n_samples": 23.0, | |
| "_scorer/specialty_derm_score": 0.6271, | |
| "_scorer/specialty_derm_std": 0.121, | |
| "_scorer/specialty_derm_n_samples": 14.0, | |
| "_scorer/specialty_emergency_score": 0.7046, | |
| "_scorer/specialty_emergency_std": 0.1229, | |
| "_scorer/specialty_emergency_n_samples": 8.0, | |
| "_scorer/specialty_endo_score": 0.7143, | |
| "_scorer/specialty_endo_std": 0.1406, | |
| "_scorer/specialty_endo_n_samples": 9.0, | |
| "_scorer/specialty_genetics_score": 0.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.4533, | |
| "_scorer/specialty_heme/onc_std": 0.1513, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.3333, | |
| "_scorer/specialty_medicine_std": 0.1901, | |
| "_scorer/specialty_medicine_n_samples": 6.0, | |
| "_scorer/specialty_nephro_score": 0.8889, | |
| "_scorer/specialty_nephro_std": 0.1006, | |
| "_scorer/specialty_nephro_n_samples": 9.0, | |
| "_scorer/specialty_neuro_score": 0.7397, | |
| "_scorer/specialty_neuro_std": 0.0941, | |
| "_scorer/specialty_neuro_n_samples": 15.0, | |
| "_scorer/specialty_neurosurg_score": 0.6667, | |
| "_scorer/specialty_neurosurg_std": 0.1924, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.6429, | |
| "_scorer/specialty_obgyn_std": 0.1007, | |
| "_scorer/specialty_obgyn_n_samples": 18.0, | |
| "_scorer/specialty_ophtho_score": 0.52, | |
| "_scorer/specialty_ophtho_std": 0.1519, | |
| "_scorer/specialty_ophtho_n_samples": 9.0, | |
| "_scorer/specialty_ortho_score": 0.6239, | |
| "_scorer/specialty_ortho_std": 0.1343, | |
| "_scorer/specialty_ortho_n_samples": 13.0, | |
| "_scorer/specialty_other_score": 0.5, | |
| "_scorer/specialty_other_std": 0.2045, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_path_score": 0.3333, | |
| "_scorer/specialty_path_std": 0.2679, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.3277, | |
| "_scorer/specialty_peds_std": 0.109, | |
| "_scorer/specialty_peds_n_samples": 14.0, | |
| "_scorer/specialty_primary_score": 0.8676, | |
| "_scorer/specialty_primary_std": 0.1125, | |
| "_scorer/specialty_primary_n_samples": 4.0, | |
| "_scorer/specialty_psychiatry_score": 0.572, | |
| "_scorer/specialty_psychiatry_std": 0.1582, | |
| "_scorer/specialty_psychiatry_n_samples": 7.0, | |
| "_scorer/specialty_pulm_score": 0.75, | |
| "_scorer/specialty_pulm_std": 0.1573, | |
| "_scorer/specialty_pulm_n_samples": 6.0, | |
| "_scorer/specialty_rads_score": 0.6429, | |
| "_scorer/specialty_rads_std": 0.1715, | |
| "_scorer/specialty_rads_n_samples": 7.0, | |
| "_scorer/specialty_rheum_score": 0.6, | |
| "_scorer/specialty_rheum_std": 0.2161, | |
| "_scorer/specialty_rheum_n_samples": 5.0, | |
| "_scorer/specialty_surgery_score": 0.9, | |
| "_scorer/specialty_surgery_std": 0.0945, | |
| "_scorer/specialty_surgery_n_samples": 10.0, | |
| "_scorer/specialty_uro_score": 0.75, | |
| "_scorer/specialty_uro_std": 0.2109, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7806, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0351, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 84.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.5066, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0618, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.5109, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0501, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, | |
| "healthbench_professional_score/mean": 0.51, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4864 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T16-37-37-00-00_healthbench-professional-writing_joZQNJ9ZAZidpUaSpEqyvS.eval", | |
| "task": "healthbench_professional_writing", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 142, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 56, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T16:37:37+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1136, | |
| "total_samples": 1136, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.3596, | |
| "_scorer/std": 0.0455, | |
| "_scorer/criteria_met_rate": 0.5657, | |
| "_scorer/use_case_writing_score": 0.5245, | |
| "_scorer/use_case_writing_std": 0.0384, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_red_teaming_score": 0.3623, | |
| "_scorer/type_red_teaming_std": 0.0488, | |
| "_scorer/type_red_teaming_n_samples": 91.0, | |
| "_scorer/type_good_faith_score": 0.8139, | |
| "_scorer/type_good_faith_std": 0.0425, | |
| "_scorer/type_good_faith_n_samples": 51.0, | |
| "_scorer/difficulty_difficult_score": 0.3903, | |
| "_scorer/difficulty_difficult_std": 0.0486, | |
| "_scorer/difficulty_difficult_n_samples": 96.0, | |
| "_scorer/difficulty_typical_score": 0.8046, | |
| "_scorer/difficulty_typical_std": 0.0437, | |
| "_scorer/difficulty_typical_n_samples": 46.0, | |
| "_scorer/specialty_ent_score": 0.5, | |
| "_scorer/specialty_ent_std": 0.3567, | |
| "_scorer/specialty_ent_n_samples": 2.0, | |
| "_scorer/specialty_gi_score": 0.4, | |
| "_scorer/specialty_gi_std": 0.2175, | |
| "_scorer/specialty_gi_n_samples": 5.0, | |
| "_scorer/specialty_id_score": 0.275, | |
| "_scorer/specialty_id_std": 0.1196, | |
| "_scorer/specialty_id_n_samples": 8.0, | |
| "_scorer/specialty_anesthesia_score": 0.3061, | |
| "_scorer/specialty_anesthesia_std": 0.1608, | |
| "_scorer/specialty_anesthesia_n_samples": 4.0, | |
| "_scorer/specialty_cards_score": 0.6667, | |
| "_scorer/specialty_cards_std": 0.1905, | |
| "_scorer/specialty_cards_n_samples": 6.0, | |
| "_scorer/specialty_derm_score": 0.3151, | |
| "_scorer/specialty_derm_std": 0.1462, | |
| "_scorer/specialty_derm_n_samples": 7.0, | |
| "_scorer/specialty_emergency_score": 0.5089, | |
| "_scorer/specialty_emergency_std": 0.1786, | |
| "_scorer/specialty_emergency_n_samples": 7.0, | |
| "_scorer/specialty_endo_score": 0.5474, | |
| "_scorer/specialty_endo_std": 0.2038, | |
| "_scorer/specialty_endo_n_samples": 5.0, | |
| "_scorer/specialty_genetics_score": 1.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.5169, | |
| "_scorer/specialty_heme/onc_std": 0.14, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.3333, | |
| "_scorer/specialty_medicine_std": 0.2712, | |
| "_scorer/specialty_medicine_n_samples": 3.0, | |
| "_scorer/specialty_nephro_score": 1.0, | |
| "_scorer/specialty_nephro_std": 0.0, | |
| "_scorer/specialty_nephro_n_samples": 6.0, | |
| "_scorer/specialty_neuro_score": 0.6111, | |
| "_scorer/specialty_neuro_std": 0.1281, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.25, | |
| "_scorer/specialty_neurosurg_std": 0.213, | |
| "_scorer/specialty_neurosurg_n_samples": 4.0, | |
| "_scorer/specialty_obgyn_score": 0.6397, | |
| "_scorer/specialty_obgyn_std": 0.1218, | |
| "_scorer/specialty_obgyn_n_samples": 12.0, | |
| "_scorer/specialty_ophtho_score": 0.475, | |
| "_scorer/specialty_ophtho_std": 0.1995, | |
| "_scorer/specialty_ophtho_n_samples": 5.0, | |
| "_scorer/specialty_ortho_score": 0.2778, | |
| "_scorer/specialty_ortho_std": 0.2099, | |
| "_scorer/specialty_ortho_n_samples": 4.0, | |
| "_scorer/specialty_other_score": 0.4167, | |
| "_scorer/specialty_other_std": 0.1612, | |
| "_scorer/specialty_other_n_samples": 8.0, | |
| "_scorer/specialty_peds_score": 0.731, | |
| "_scorer/specialty_peds_std": 0.132, | |
| "_scorer/specialty_peds_n_samples": 8.0, | |
| "_scorer/specialty_primary_score": 0.6667, | |
| "_scorer/specialty_primary_std": 0.2857, | |
| "_scorer/specialty_primary_n_samples": 3.0, | |
| "_scorer/specialty_psychiatry_score": 0.6278, | |
| "_scorer/specialty_psychiatry_std": 0.1312, | |
| "_scorer/specialty_psychiatry_n_samples": 12.0, | |
| "_scorer/specialty_pulm_score": 1.0, | |
| "_scorer/specialty_pulm_std": 0.0, | |
| "_scorer/specialty_pulm_n_samples": 2.0, | |
| "_scorer/specialty_rads_score": 1.0, | |
| "_scorer/specialty_rads_std": 0.0, | |
| "_scorer/specialty_rads_n_samples": 1.0, | |
| "_scorer/specialty_rheum_score": 0.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 2.0, | |
| "_scorer/specialty_surgery_score": 0.25, | |
| "_scorer/specialty_surgery_std": 0.2109, | |
| "_scorer/specialty_surgery_n_samples": 4.0, | |
| "_scorer/specialty_uro_score": 0.5, | |
| "_scorer/specialty_uro_std": 0.2527, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.3623, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0488, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.9, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0875, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.8046, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0453, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 46.0, | |
| "healthbench_professional_score/mean": 0.4062, | |
| "healthbench_professional_score_length_adjusted/mean": 0.3596 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T16-37-58-00-00_healthbench-professional-research_3Vd4jhz6oiMQTsC2Mhw849.eval", | |
| "task": "healthbench_professional_research", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 147, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 58, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T16:37:58+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1176, | |
| "total_samples": 1176, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.5792, | |
| "_scorer/std": 0.0326, | |
| "_scorer/criteria_met_rate": 0.6963, | |
| "_scorer/use_case_research_score": 0.7055, | |
| "_scorer/use_case_research_std": 0.0298, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/type_red_teaming_score": 1.0, | |
| "_scorer/type_red_teaming_std": 0.0, | |
| "_scorer/type_red_teaming_n_samples": 7.0, | |
| "_scorer/type_good_faith_score": 0.6908, | |
| "_scorer/type_good_faith_std": 0.0319, | |
| "_scorer/type_good_faith_n_samples": 140.0, | |
| "_scorer/difficulty_difficult_score": 0.5952, | |
| "_scorer/difficulty_difficult_std": 0.1043, | |
| "_scorer/difficulty_difficult_n_samples": 21.0, | |
| "_scorer/difficulty_typical_score": 0.7239, | |
| "_scorer/difficulty_typical_std": 0.0317, | |
| "_scorer/difficulty_typical_n_samples": 126.0, | |
| "_scorer/specialty_ent_score": 1.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 1.0, | |
| "_scorer/specialty_gi_score": 0.4165, | |
| "_scorer/specialty_gi_std": 0.1281, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.7302, | |
| "_scorer/specialty_id_std": 0.0844, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_anesthesia_score": 0.8214, | |
| "_scorer/specialty_anesthesia_std": 0.1203, | |
| "_scorer/specialty_anesthesia_n_samples": 8.0, | |
| "_scorer/specialty_cards_score": 0.748, | |
| "_scorer/specialty_cards_std": 0.0905, | |
| "_scorer/specialty_cards_n_samples": 14.0, | |
| "_scorer/specialty_derm_score": 1.0, | |
| "_scorer/specialty_derm_std": 0.0, | |
| "_scorer/specialty_derm_n_samples": 4.0, | |
| "_scorer/specialty_emergency_score": 1.0, | |
| "_scorer/specialty_emergency_std": 0.0, | |
| "_scorer/specialty_emergency_n_samples": 1.0, | |
| "_scorer/specialty_endo_score": 1.0, | |
| "_scorer/specialty_endo_std": 0.0, | |
| "_scorer/specialty_endo_n_samples": 6.0, | |
| "_scorer/specialty_genetics_score": 1.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 2.0, | |
| "_scorer/specialty_heme/onc_score": 0.5922, | |
| "_scorer/specialty_heme/onc_std": 0.1207, | |
| "_scorer/specialty_heme/onc_n_samples": 7.0, | |
| "_scorer/specialty_medicine_score": 0.5476, | |
| "_scorer/specialty_medicine_std": 0.165, | |
| "_scorer/specialty_medicine_n_samples": 8.0, | |
| "_scorer/specialty_nephro_score": 0.6752, | |
| "_scorer/specialty_nephro_std": 0.1315, | |
| "_scorer/specialty_nephro_n_samples": 7.0, | |
| "_scorer/specialty_neuro_score": 0.7423, | |
| "_scorer/specialty_neuro_std": 0.075, | |
| "_scorer/specialty_neuro_n_samples": 19.0, | |
| "_scorer/specialty_neurosurg_score": 0.0667, | |
| "_scorer/specialty_neurosurg_std": 0.0, | |
| "_scorer/specialty_neurosurg_n_samples": 1.0, | |
| "_scorer/specialty_obgyn_score": 0.697, | |
| "_scorer/specialty_obgyn_std": 0.0934, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 1.0, | |
| "_scorer/specialty_ophtho_std": 0.0, | |
| "_scorer/specialty_ophtho_n_samples": 4.0, | |
| "_scorer/specialty_ortho_score": 0.2036, | |
| "_scorer/specialty_ortho_std": 0.0827, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.5625, | |
| "_scorer/specialty_other_std": 0.1729, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_peds_score": 0.7619, | |
| "_scorer/specialty_peds_std": 0.1562, | |
| "_scorer/specialty_peds_n_samples": 6.0, | |
| "_scorer/specialty_psychiatry_score": 0.8125, | |
| "_scorer/specialty_psychiatry_std": 0.1149, | |
| "_scorer/specialty_psychiatry_n_samples": 10.0, | |
| "_scorer/specialty_pulm_score": 0.6198, | |
| "_scorer/specialty_pulm_std": 0.1396, | |
| "_scorer/specialty_pulm_n_samples": 4.0, | |
| "_scorer/specialty_rheum_score": 1.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 1.0, | |
| "_scorer/specialty_surgery_score": 0.8333, | |
| "_scorer/specialty_surgery_std": 0.1358, | |
| "_scorer/specialty_surgery_n_samples": 3.0, | |
| "_scorer/specialty_uro_score": 0.7, | |
| "_scorer/specialty_uro_std": 0.1781, | |
| "_scorer/specialty_uro_n_samples": 5.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 1.0, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7239, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0311, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 126.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3929, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.1233, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, | |
| "healthbench_professional_score/mean": 0.6799, | |
| "healthbench_professional_score_length_adjusted/mean": 0.5792 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T16-38-22-00-00_healthbench-professional-red-teaming_Xund3mwQvSqTJfxno7icGe.eval", | |
| "task": "healthbench_professional_red_teaming", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 191, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 76, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T16:38:22+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1528, | |
| "total_samples": 1528, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2819, | |
| "_scorer/std": 0.0444, | |
| "_scorer/criteria_met_rate": 0.4959, | |
| "_scorer/use_case_consult_score": 0.5109, | |
| "_scorer/use_case_consult_std": 0.0509, | |
| "_scorer/use_case_consult_n_samples": 93.0, | |
| "_scorer/use_case_research_score": 1.0, | |
| "_scorer/use_case_research_std": 0.0, | |
| "_scorer/use_case_research_n_samples": 7.0, | |
| "_scorer/use_case_writing_score": 0.3623, | |
| "_scorer/use_case_writing_std": 0.0488, | |
| "_scorer/use_case_writing_n_samples": 91.0, | |
| "_scorer/type_red_teaming_score": 0.458, | |
| "_scorer/type_red_teaming_std": 0.034, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_difficult_score": 0.458, | |
| "_scorer/difficulty_difficult_std": 0.0349, | |
| "_scorer/difficulty_difficult_n_samples": 191.0, | |
| "_scorer/specialty_ent_score": 0.75, | |
| "_scorer/specialty_ent_std": 0.217, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.25, | |
| "_scorer/specialty_gi_std": 0.1523, | |
| "_scorer/specialty_gi_n_samples": 8.0, | |
| "_scorer/specialty_id_score": 0.32, | |
| "_scorer/specialty_id_std": 0.1214, | |
| "_scorer/specialty_id_n_samples": 10.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 1.0, | |
| "_scorer/specialty_anesthesia_score": 0.5238, | |
| "_scorer/specialty_anesthesia_std": 0.1767, | |
| "_scorer/specialty_anesthesia_n_samples": 7.0, | |
| "_scorer/specialty_cards_score": 0.303, | |
| "_scorer/specialty_cards_std": 0.1237, | |
| "_scorer/specialty_cards_n_samples": 11.0, | |
| "_scorer/specialty_derm_score": 0.3714, | |
| "_scorer/specialty_derm_std": 0.1343, | |
| "_scorer/specialty_derm_n_samples": 11.0, | |
| "_scorer/specialty_emergency_score": 0.5966, | |
| "_scorer/specialty_emergency_std": 0.1383, | |
| "_scorer/specialty_emergency_n_samples": 11.0, | |
| "_scorer/specialty_endo_score": 0.5429, | |
| "_scorer/specialty_endo_std": 0.1471, | |
| "_scorer/specialty_endo_n_samples": 10.0, | |
| "_scorer/specialty_genetics_score": 0.6667, | |
| "_scorer/specialty_genetics_std": 0.2765, | |
| "_scorer/specialty_genetics_n_samples": 3.0, | |
| "_scorer/specialty_heme/onc_score": 0.4067, | |
| "_scorer/specialty_heme/onc_std": 0.138, | |
| "_scorer/specialty_heme/onc_n_samples": 11.0, | |
| "_scorer/specialty_medicine_score": 0.2857, | |
| "_scorer/specialty_medicine_std": 0.1691, | |
| "_scorer/specialty_medicine_n_samples": 7.0, | |
| "_scorer/specialty_nephro_score": 0.75, | |
| "_scorer/specialty_nephro_std": 0.2217, | |
| "_scorer/specialty_nephro_n_samples": 4.0, | |
| "_scorer/specialty_neuro_score": 0.6667, | |
| "_scorer/specialty_neuro_std": 0.1524, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.3333, | |
| "_scorer/specialty_neurosurg_std": 0.1956, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.5215, | |
| "_scorer/specialty_obgyn_std": 0.1434, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 0.3333, | |
| "_scorer/specialty_ophtho_std": 0.2704, | |
| "_scorer/specialty_ophtho_n_samples": 3.0, | |
| "_scorer/specialty_ortho_score": 0.3016, | |
| "_scorer/specialty_ortho_std": 0.1682, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.2222, | |
| "_scorer/specialty_other_std": 0.1392, | |
| "_scorer/specialty_other_n_samples": 9.0, | |
| "_scorer/specialty_path_score": 1.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 1.0, | |
| "_scorer/specialty_peds_score": 0.3407, | |
| "_scorer/specialty_peds_std": 0.1202, | |
| "_scorer/specialty_peds_n_samples": 10.0, | |
| "_scorer/specialty_primary_score": 0.8, | |
| "_scorer/specialty_primary_std": 0.1815, | |
| "_scorer/specialty_primary_n_samples": 5.0, | |
| "_scorer/specialty_psychiatry_score": 0.3333, | |
| "_scorer/specialty_psychiatry_std": 0.1553, | |
| "_scorer/specialty_psychiatry_n_samples": 9.0, | |
| "_scorer/specialty_pulm_score": 0.8, | |
| "_scorer/specialty_pulm_std": 0.1838, | |
| "_scorer/specialty_pulm_n_samples": 5.0, | |
| "_scorer/specialty_rads_score": 0.8, | |
| "_scorer/specialty_rads_std": 0.1706, | |
| "_scorer/specialty_rads_n_samples": 5.0, | |
| "_scorer/specialty_rheum_score": 0.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 4.0, | |
| "_scorer/specialty_surgery_score": 0.5714, | |
| "_scorer/specialty_surgery_std": 0.1882, | |
| "_scorer/specialty_surgery_n_samples": 7.0, | |
| "_scorer/specialty_uro_score": 0.6667, | |
| "_scorer/specialty_uro_std": 0.2724, | |
| "_scorer/specialty_uro_n_samples": 3.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.458, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0353, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.2992, | |
| "healthbench_professional_score_length_adjusted/mean": 0.2819 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T16-38-52-00-00_healthbench-professional-physician-baseline_XADueRF5xwkSmDPWc4LtNf.eval", | |
| "task": "healthbench_professional_physician_baseline", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T16:38:52+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4387, | |
| "_scorer/std": 0.0244, | |
| "_scorer/criteria_met_rate": 0.5044, | |
| "_scorer/use_case_consult_score": 0.4843, | |
| "_scorer/use_case_consult_std": 0.0284, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.59, | |
| "_scorer/use_case_research_std": 0.0312, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.449, | |
| "_scorer/use_case_writing_std": 0.0356, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.5561, | |
| "_scorer/type_good_faith_std": 0.0211, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.4137, | |
| "_scorer/type_red_teaming_std": 0.0327, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.5803, | |
| "_scorer/difficulty_typical_std": 0.023, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.432, | |
| "_scorer/difficulty_difficult_std": 0.0285, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.4857, | |
| "_scorer/specialty_ent_std": 0.175, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.5427, | |
| "_scorer/specialty_gi_std": 0.0933, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.5113, | |
| "_scorer/specialty_id_std": 0.0732, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.6229, | |
| "_scorer/specialty_anesthesia_std": 0.087, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.4092, | |
| "_scorer/specialty_cards_std": 0.0625, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.6483, | |
| "_scorer/specialty_derm_std": 0.0798, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.4549, | |
| "_scorer/specialty_emergency_std": 0.1188, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5268, | |
| "_scorer/specialty_endo_std": 0.0835, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.7955, | |
| "_scorer/specialty_genetics_std": 0.105, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.5019, | |
| "_scorer/specialty_heme/onc_std": 0.0808, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.6713, | |
| "_scorer/specialty_medicine_std": 0.098, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.4311, | |
| "_scorer/specialty_nephro_std": 0.0796, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.5592, | |
| "_scorer/specialty_neuro_std": 0.0605, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.3242, | |
| "_scorer/specialty_neurosurg_std": 0.1175, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.4791, | |
| "_scorer/specialty_obgyn_std": 0.0641, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.3167, | |
| "_scorer/specialty_ophtho_std": 0.0976, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.3507, | |
| "_scorer/specialty_ortho_std": 0.0825, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.3541, | |
| "_scorer/specialty_other_std": 0.0835, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.6667, | |
| "_scorer/specialty_path_std": 0.2716, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.5363, | |
| "_scorer/specialty_peds_std": 0.0853, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.563, | |
| "_scorer/specialty_primary_std": 0.1627, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.5115, | |
| "_scorer/specialty_psychiatry_std": 0.0697, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.5848, | |
| "_scorer/specialty_pulm_std": 0.1138, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.3787, | |
| "_scorer/specialty_rads_std": 0.1538, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.6964, | |
| "_scorer/specialty_rheum_std": 0.1524, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.498, | |
| "_scorer/specialty_surgery_std": 0.1142, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.6175, | |
| "_scorer/specialty_uro_std": 0.1068, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.5803, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0233, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.4768, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0526, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.4137, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0324, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.4429, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4387 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-5.4": 17204312 | |
| }, | |
| "error": null, | |
| "solver": "physician_baseline_solver" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T17-19-14-00-00_healthbench-professional-consult_hZgDXu6oT2MuZrn4L7msXN.eval", | |
| "task": "healthbench_professional_consult", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 236, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 94, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T17:19:14+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1888, | |
| "total_samples": 1888, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4704, | |
| "_scorer/std": 0.035, | |
| "_scorer/criteria_met_rate": 0.6144, | |
| "_scorer/use_case_consult_score": 0.6051, | |
| "_scorer/use_case_consult_std": 0.0281, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/type_good_faith_score": 0.6569, | |
| "_scorer/type_good_faith_std": 0.0351, | |
| "_scorer/type_good_faith_n_samples": 143.0, | |
| "_scorer/type_red_teaming_score": 0.5254, | |
| "_scorer/type_red_teaming_std": 0.0477, | |
| "_scorer/type_red_teaming_n_samples": 93.0, | |
| "_scorer/difficulty_typical_score": 0.7495, | |
| "_scorer/difficulty_typical_std": 0.0356, | |
| "_scorer/difficulty_typical_n_samples": 84.0, | |
| "_scorer/difficulty_difficult_score": 0.5253, | |
| "_scorer/difficulty_difficult_std": 0.0375, | |
| "_scorer/difficulty_difficult_n_samples": 152.0, | |
| "_scorer/specialty_ent_score": 0.6937, | |
| "_scorer/specialty_ent_std": 0.2083, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.4459, | |
| "_scorer/specialty_gi_std": 0.1624, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.75, | |
| "_scorer/specialty_id_std": 0.1559, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_allergy_score": 0.5, | |
| "_scorer/specialty_allergy_std": 0.3575, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.8275, | |
| "_scorer/specialty_anesthesia_std": 0.0941, | |
| "_scorer/specialty_anesthesia_n_samples": 10.0, | |
| "_scorer/specialty_cards_score": 0.4499, | |
| "_scorer/specialty_cards_std": 0.088, | |
| "_scorer/specialty_cards_n_samples": 23.0, | |
| "_scorer/specialty_derm_score": 0.7283, | |
| "_scorer/specialty_derm_std": 0.111, | |
| "_scorer/specialty_derm_n_samples": 14.0, | |
| "_scorer/specialty_emergency_score": 0.6838, | |
| "_scorer/specialty_emergency_std": 0.1457, | |
| "_scorer/specialty_emergency_n_samples": 8.0, | |
| "_scorer/specialty_endo_score": 0.7143, | |
| "_scorer/specialty_endo_std": 0.1396, | |
| "_scorer/specialty_endo_n_samples": 9.0, | |
| "_scorer/specialty_genetics_score": 0.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.406, | |
| "_scorer/specialty_heme/onc_std": 0.1395, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.4993, | |
| "_scorer/specialty_medicine_std": 0.1594, | |
| "_scorer/specialty_medicine_n_samples": 6.0, | |
| "_scorer/specialty_nephro_score": 0.6942, | |
| "_scorer/specialty_nephro_std": 0.1144, | |
| "_scorer/specialty_nephro_n_samples": 9.0, | |
| "_scorer/specialty_neuro_score": 0.7375, | |
| "_scorer/specialty_neuro_std": 0.0976, | |
| "_scorer/specialty_neuro_n_samples": 15.0, | |
| "_scorer/specialty_neurosurg_score": 0.7588, | |
| "_scorer/specialty_neurosurg_std": 0.1038, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.5495, | |
| "_scorer/specialty_obgyn_std": 0.0928, | |
| "_scorer/specialty_obgyn_n_samples": 18.0, | |
| "_scorer/specialty_ophtho_score": 0.4089, | |
| "_scorer/specialty_ophtho_std": 0.15, | |
| "_scorer/specialty_ophtho_n_samples": 9.0, | |
| "_scorer/specialty_ortho_score": 0.8103, | |
| "_scorer/specialty_ortho_std": 0.1004, | |
| "_scorer/specialty_ortho_n_samples": 13.0, | |
| "_scorer/specialty_other_score": 0.6667, | |
| "_scorer/specialty_other_std": 0.1911, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_path_score": 0.1905, | |
| "_scorer/specialty_path_std": 0.1574, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.2979, | |
| "_scorer/specialty_peds_std": 0.1081, | |
| "_scorer/specialty_peds_n_samples": 14.0, | |
| "_scorer/specialty_primary_score": 0.6176, | |
| "_scorer/specialty_primary_std": 0.2146, | |
| "_scorer/specialty_primary_n_samples": 4.0, | |
| "_scorer/specialty_psychiatry_score": 0.6392, | |
| "_scorer/specialty_psychiatry_std": 0.1309, | |
| "_scorer/specialty_psychiatry_n_samples": 7.0, | |
| "_scorer/specialty_pulm_score": 0.5, | |
| "_scorer/specialty_pulm_std": 0.1634, | |
| "_scorer/specialty_pulm_n_samples": 6.0, | |
| "_scorer/specialty_rads_score": 0.7101, | |
| "_scorer/specialty_rads_std": 0.1344, | |
| "_scorer/specialty_rads_n_samples": 7.0, | |
| "_scorer/specialty_rheum_score": 0.6, | |
| "_scorer/specialty_rheum_std": 0.2164, | |
| "_scorer/specialty_rheum_n_samples": 5.0, | |
| "_scorer/specialty_surgery_score": 0.8, | |
| "_scorer/specialty_surgery_std": 0.1269, | |
| "_scorer/specialty_surgery_n_samples": 10.0, | |
| "_scorer/specialty_uro_score": 0.75, | |
| "_scorer/specialty_uro_std": 0.2207, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7495, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0359, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 84.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.525, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0609, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.5254, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0474, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, | |
| "healthbench_professional_score/mean": 0.4906, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4704 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T17-19-42-00-00_healthbench-professional-writing_haJJesaYyVNeNRLHnAi8HD.eval", | |
| "task": "healthbench_professional_writing", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 142, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 56, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T17:19:42+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1136, | |
| "total_samples": 1136, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.3615, | |
| "_scorer/std": 0.0505, | |
| "_scorer/criteria_met_rate": 0.5814, | |
| "_scorer/use_case_writing_score": 0.5091, | |
| "_scorer/use_case_writing_std": 0.0382, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_red_teaming_score": 0.348, | |
| "_scorer/type_red_teaming_std": 0.0488, | |
| "_scorer/type_red_teaming_n_samples": 91.0, | |
| "_scorer/type_good_faith_score": 0.7967, | |
| "_scorer/type_good_faith_std": 0.0431, | |
| "_scorer/type_good_faith_n_samples": 51.0, | |
| "_scorer/difficulty_difficult_score": 0.3767, | |
| "_scorer/difficulty_difficult_std": 0.0452, | |
| "_scorer/difficulty_difficult_n_samples": 96.0, | |
| "_scorer/difficulty_typical_score": 0.7854, | |
| "_scorer/difficulty_typical_std": 0.0478, | |
| "_scorer/difficulty_typical_n_samples": 46.0, | |
| "_scorer/specialty_ent_score": 0.5, | |
| "_scorer/specialty_ent_std": 0.3499, | |
| "_scorer/specialty_ent_n_samples": 2.0, | |
| "_scorer/specialty_gi_score": 0.5059, | |
| "_scorer/specialty_gi_std": 0.1975, | |
| "_scorer/specialty_gi_n_samples": 5.0, | |
| "_scorer/specialty_id_score": 0.4, | |
| "_scorer/specialty_id_std": 0.1399, | |
| "_scorer/specialty_id_n_samples": 8.0, | |
| "_scorer/specialty_anesthesia_score": 0.5, | |
| "_scorer/specialty_anesthesia_std": 0.2552, | |
| "_scorer/specialty_anesthesia_n_samples": 4.0, | |
| "_scorer/specialty_cards_score": 0.625, | |
| "_scorer/specialty_cards_std": 0.1838, | |
| "_scorer/specialty_cards_n_samples": 6.0, | |
| "_scorer/specialty_derm_score": 0.3151, | |
| "_scorer/specialty_derm_std": 0.1496, | |
| "_scorer/specialty_derm_n_samples": 7.0, | |
| "_scorer/specialty_emergency_score": 0.2857, | |
| "_scorer/specialty_emergency_std": 0.1775, | |
| "_scorer/specialty_emergency_n_samples": 7.0, | |
| "_scorer/specialty_endo_score": 0.3474, | |
| "_scorer/specialty_endo_std": 0.1936, | |
| "_scorer/specialty_endo_n_samples": 5.0, | |
| "_scorer/specialty_genetics_score": 0.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.5111, | |
| "_scorer/specialty_heme/onc_std": 0.1422, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.6667, | |
| "_scorer/specialty_medicine_std": 0.272, | |
| "_scorer/specialty_medicine_n_samples": 3.0, | |
| "_scorer/specialty_nephro_score": 0.9053, | |
| "_scorer/specialty_nephro_std": 0.054, | |
| "_scorer/specialty_nephro_n_samples": 6.0, | |
| "_scorer/specialty_neuro_score": 0.6685, | |
| "_scorer/specialty_neuro_std": 0.1344, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.25, | |
| "_scorer/specialty_neurosurg_std": 0.2219, | |
| "_scorer/specialty_neurosurg_n_samples": 4.0, | |
| "_scorer/specialty_obgyn_score": 0.5687, | |
| "_scorer/specialty_obgyn_std": 0.1221, | |
| "_scorer/specialty_obgyn_n_samples": 12.0, | |
| "_scorer/specialty_ophtho_score": 0.475, | |
| "_scorer/specialty_ophtho_std": 0.2006, | |
| "_scorer/specialty_ophtho_n_samples": 5.0, | |
| "_scorer/specialty_ortho_score": 0.2778, | |
| "_scorer/specialty_ortho_std": 0.2095, | |
| "_scorer/specialty_ortho_n_samples": 4.0, | |
| "_scorer/specialty_other_score": 0.4792, | |
| "_scorer/specialty_other_std": 0.152, | |
| "_scorer/specialty_other_n_samples": 8.0, | |
| "_scorer/specialty_peds_score": 0.625, | |
| "_scorer/specialty_peds_std": 0.1493, | |
| "_scorer/specialty_peds_n_samples": 8.0, | |
| "_scorer/specialty_primary_score": 0.6667, | |
| "_scorer/specialty_primary_std": 0.2683, | |
| "_scorer/specialty_primary_n_samples": 3.0, | |
| "_scorer/specialty_psychiatry_score": 0.6808, | |
| "_scorer/specialty_psychiatry_std": 0.1225, | |
| "_scorer/specialty_psychiatry_n_samples": 12.0, | |
| "_scorer/specialty_pulm_score": 1.0, | |
| "_scorer/specialty_pulm_std": 0.0, | |
| "_scorer/specialty_pulm_n_samples": 2.0, | |
| "_scorer/specialty_rads_score": 1.0, | |
| "_scorer/specialty_rads_std": 0.0, | |
| "_scorer/specialty_rads_n_samples": 1.0, | |
| "_scorer/specialty_rheum_score": 0.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 2.0, | |
| "_scorer/specialty_surgery_score": 0.25, | |
| "_scorer/specialty_surgery_std": 0.2153, | |
| "_scorer/specialty_surgery_n_samples": 4.0, | |
| "_scorer/specialty_uro_score": 0.25, | |
| "_scorer/specialty_uro_std": 0.2181, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.348, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0463, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.9, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0893, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7854, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0452, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 46.0, | |
| "healthbench_professional_score/mean": 0.3947, | |
| "healthbench_professional_score_length_adjusted/mean": 0.3615 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T17-19-59-00-00_healthbench-professional-research_7RUfdNFLnH7EpAC92NeL7e.eval", | |
| "task": "healthbench_professional_research", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 147, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 58, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T17:19:59+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1176, | |
| "total_samples": 1176, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.6107, | |
| "_scorer/std": 0.0317, | |
| "_scorer/criteria_met_rate": 0.6757, | |
| "_scorer/use_case_research_score": 0.6739, | |
| "_scorer/use_case_research_std": 0.0313, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/type_red_teaming_score": 1.0, | |
| "_scorer/type_red_teaming_std": 0.0, | |
| "_scorer/type_red_teaming_n_samples": 7.0, | |
| "_scorer/type_good_faith_score": 0.6576, | |
| "_scorer/type_good_faith_std": 0.0331, | |
| "_scorer/type_good_faith_n_samples": 140.0, | |
| "_scorer/difficulty_difficult_score": 0.5476, | |
| "_scorer/difficulty_difficult_std": 0.1061, | |
| "_scorer/difficulty_difficult_n_samples": 21.0, | |
| "_scorer/difficulty_typical_score": 0.6949, | |
| "_scorer/difficulty_typical_std": 0.0316, | |
| "_scorer/difficulty_typical_n_samples": 126.0, | |
| "_scorer/specialty_ent_score": 1.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 1.0, | |
| "_scorer/specialty_gi_score": 0.3108, | |
| "_scorer/specialty_gi_std": 0.1035, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.7857, | |
| "_scorer/specialty_id_std": 0.0927, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_anesthesia_score": 0.8214, | |
| "_scorer/specialty_anesthesia_std": 0.0802, | |
| "_scorer/specialty_anesthesia_n_samples": 8.0, | |
| "_scorer/specialty_cards_score": 0.7769, | |
| "_scorer/specialty_cards_std": 0.0774, | |
| "_scorer/specialty_cards_n_samples": 14.0, | |
| "_scorer/specialty_derm_score": 0.75, | |
| "_scorer/specialty_derm_std": 0.224, | |
| "_scorer/specialty_derm_n_samples": 4.0, | |
| "_scorer/specialty_emergency_score": 1.0, | |
| "_scorer/specialty_emergency_std": 0.0, | |
| "_scorer/specialty_emergency_n_samples": 1.0, | |
| "_scorer/specialty_endo_score": 0.94, | |
| "_scorer/specialty_endo_std": 0.0544, | |
| "_scorer/specialty_endo_n_samples": 6.0, | |
| "_scorer/specialty_genetics_score": 0.8409, | |
| "_scorer/specialty_genetics_std": 0.1137, | |
| "_scorer/specialty_genetics_n_samples": 2.0, | |
| "_scorer/specialty_heme/onc_score": 0.5498, | |
| "_scorer/specialty_heme/onc_std": 0.1444, | |
| "_scorer/specialty_heme/onc_n_samples": 7.0, | |
| "_scorer/specialty_medicine_score": 0.5817, | |
| "_scorer/specialty_medicine_std": 0.1606, | |
| "_scorer/specialty_medicine_n_samples": 8.0, | |
| "_scorer/specialty_nephro_score": 0.8031, | |
| "_scorer/specialty_nephro_std": 0.1318, | |
| "_scorer/specialty_nephro_n_samples": 7.0, | |
| "_scorer/specialty_neuro_score": 0.7282, | |
| "_scorer/specialty_neuro_std": 0.0731, | |
| "_scorer/specialty_neuro_n_samples": 19.0, | |
| "_scorer/specialty_neurosurg_score": 0.0667, | |
| "_scorer/specialty_neurosurg_std": 0.0, | |
| "_scorer/specialty_neurosurg_n_samples": 1.0, | |
| "_scorer/specialty_obgyn_score": 0.5, | |
| "_scorer/specialty_obgyn_std": 0.1406, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 1.0, | |
| "_scorer/specialty_ophtho_std": 0.0, | |
| "_scorer/specialty_ophtho_n_samples": 4.0, | |
| "_scorer/specialty_ortho_score": 0.3702, | |
| "_scorer/specialty_ortho_std": 0.1636, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.6267, | |
| "_scorer/specialty_other_std": 0.1436, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_peds_score": 0.6613, | |
| "_scorer/specialty_peds_std": 0.1598, | |
| "_scorer/specialty_peds_n_samples": 6.0, | |
| "_scorer/specialty_psychiatry_score": 0.675, | |
| "_scorer/specialty_psychiatry_std": 0.1402, | |
| "_scorer/specialty_psychiatry_n_samples": 10.0, | |
| "_scorer/specialty_pulm_score": 0.5401, | |
| "_scorer/specialty_pulm_std": 0.1804, | |
| "_scorer/specialty_pulm_n_samples": 4.0, | |
| "_scorer/specialty_rheum_score": 1.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 1.0, | |
| "_scorer/specialty_surgery_score": 0.8333, | |
| "_scorer/specialty_surgery_std": 0.1347, | |
| "_scorer/specialty_surgery_n_samples": 3.0, | |
| "_scorer/specialty_uro_score": 0.5286, | |
| "_scorer/specialty_uro_std": 0.181, | |
| "_scorer/specialty_uro_n_samples": 5.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 1.0, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.6949, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0316, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 126.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3214, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.1197, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, | |
| "healthbench_professional_score/mean": 0.6464, | |
| "healthbench_professional_score_length_adjusted/mean": 0.6107 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T17-20-17-00-00_healthbench-professional-red-teaming_DLPPa6kbs9VeKS6cfPDhdv.eval", | |
| "task": "healthbench_professional_red_teaming", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 191, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 76, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T17:20:17+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1528, | |
| "total_samples": 1528, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2674, | |
| "_scorer/std": 0.0457, | |
| "_scorer/criteria_met_rate": 0.5191, | |
| "_scorer/use_case_consult_score": 0.5254, | |
| "_scorer/use_case_consult_std": 0.0469, | |
| "_scorer/use_case_consult_n_samples": 93.0, | |
| "_scorer/use_case_research_score": 1.0, | |
| "_scorer/use_case_research_std": 0.0, | |
| "_scorer/use_case_research_n_samples": 7.0, | |
| "_scorer/use_case_writing_score": 0.348, | |
| "_scorer/use_case_writing_std": 0.047, | |
| "_scorer/use_case_writing_n_samples": 91.0, | |
| "_scorer/type_red_teaming_score": 0.4583, | |
| "_scorer/type_red_teaming_std": 0.0328, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_difficult_score": 0.4583, | |
| "_scorer/difficulty_difficult_std": 0.0342, | |
| "_scorer/difficulty_difficult_n_samples": 191.0, | |
| "_scorer/specialty_ent_score": 0.5, | |
| "_scorer/specialty_ent_std": 0.2537, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.3235, | |
| "_scorer/specialty_gi_std": 0.1471, | |
| "_scorer/specialty_gi_n_samples": 8.0, | |
| "_scorer/specialty_id_score": 0.42, | |
| "_scorer/specialty_id_std": 0.1362, | |
| "_scorer/specialty_id_n_samples": 10.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 1.0, | |
| "_scorer/specialty_anesthesia_score": 0.5238, | |
| "_scorer/specialty_anesthesia_std": 0.172, | |
| "_scorer/specialty_anesthesia_n_samples": 7.0, | |
| "_scorer/specialty_cards_score": 0.4046, | |
| "_scorer/specialty_cards_std": 0.1274, | |
| "_scorer/specialty_cards_n_samples": 11.0, | |
| "_scorer/specialty_derm_score": 0.3714, | |
| "_scorer/specialty_derm_std": 0.1265, | |
| "_scorer/specialty_derm_n_samples": 11.0, | |
| "_scorer/specialty_emergency_score": 0.4545, | |
| "_scorer/specialty_emergency_std": 0.1481, | |
| "_scorer/specialty_emergency_n_samples": 11.0, | |
| "_scorer/specialty_endo_score": 0.4429, | |
| "_scorer/specialty_endo_std": 0.1471, | |
| "_scorer/specialty_endo_n_samples": 10.0, | |
| "_scorer/specialty_genetics_score": 0.3333, | |
| "_scorer/specialty_genetics_std": 0.2626, | |
| "_scorer/specialty_genetics_n_samples": 3.0, | |
| "_scorer/specialty_heme/onc_score": 0.361, | |
| "_scorer/specialty_heme/onc_std": 0.1306, | |
| "_scorer/specialty_heme/onc_n_samples": 11.0, | |
| "_scorer/specialty_medicine_score": 0.6471, | |
| "_scorer/specialty_medicine_std": 0.1607, | |
| "_scorer/specialty_medicine_n_samples": 7.0, | |
| "_scorer/specialty_nephro_score": 0.75, | |
| "_scorer/specialty_nephro_std": 0.2211, | |
| "_scorer/specialty_nephro_n_samples": 4.0, | |
| "_scorer/specialty_neuro_score": 0.6711, | |
| "_scorer/specialty_neuro_std": 0.1604, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.4255, | |
| "_scorer/specialty_neurosurg_std": 0.1475, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.4825, | |
| "_scorer/specialty_obgyn_std": 0.1368, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 0.3333, | |
| "_scorer/specialty_ophtho_std": 0.2679, | |
| "_scorer/specialty_ophtho_n_samples": 3.0, | |
| "_scorer/specialty_ortho_score": 0.5873, | |
| "_scorer/specialty_ortho_std": 0.178, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.3889, | |
| "_scorer/specialty_other_std": 0.1543, | |
| "_scorer/specialty_other_n_samples": 9.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 1.0, | |
| "_scorer/specialty_peds_score": 0.2641, | |
| "_scorer/specialty_peds_std": 0.1266, | |
| "_scorer/specialty_peds_n_samples": 10.0, | |
| "_scorer/specialty_primary_score": 0.6, | |
| "_scorer/specialty_primary_std": 0.2216, | |
| "_scorer/specialty_primary_n_samples": 5.0, | |
| "_scorer/specialty_psychiatry_score": 0.4967, | |
| "_scorer/specialty_psychiatry_std": 0.1586, | |
| "_scorer/specialty_psychiatry_n_samples": 9.0, | |
| "_scorer/specialty_pulm_score": 0.6, | |
| "_scorer/specialty_pulm_std": 0.2172, | |
| "_scorer/specialty_pulm_n_samples": 5.0, | |
| "_scorer/specialty_rads_score": 0.8941, | |
| "_scorer/specialty_rads_std": 0.0905, | |
| "_scorer/specialty_rads_n_samples": 5.0, | |
| "_scorer/specialty_rheum_score": 0.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 4.0, | |
| "_scorer/specialty_surgery_score": 0.4286, | |
| "_scorer/specialty_surgery_std": 0.1844, | |
| "_scorer/specialty_surgery_n_samples": 7.0, | |
| "_scorer/specialty_uro_score": 0.3333, | |
| "_scorer/specialty_uro_std": 0.2713, | |
| "_scorer/specialty_uro_n_samples": 3.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.4583, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0337, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.2828, | |
| "healthbench_professional_score_length_adjusted/mean": 0.2674 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T17-20-42-00-00_healthbench-professional-physician-baseline_98adhjPPBGUtnnuJVmWXjn.eval", | |
| "task": "healthbench_professional_physician_baseline", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T17:20:42+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4387, | |
| "_scorer/std": 0.023, | |
| "_scorer/criteria_met_rate": 0.5044, | |
| "_scorer/use_case_consult_score": 0.4843, | |
| "_scorer/use_case_consult_std": 0.0275, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.59, | |
| "_scorer/use_case_research_std": 0.0309, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.449, | |
| "_scorer/use_case_writing_std": 0.0372, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.5561, | |
| "_scorer/type_good_faith_std": 0.0209, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.4137, | |
| "_scorer/type_red_teaming_std": 0.0318, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.5803, | |
| "_scorer/difficulty_typical_std": 0.0222, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.432, | |
| "_scorer/difficulty_difficult_std": 0.0271, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.4857, | |
| "_scorer/specialty_ent_std": 0.176, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.5427, | |
| "_scorer/specialty_gi_std": 0.0962, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.5113, | |
| "_scorer/specialty_id_std": 0.0721, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.6229, | |
| "_scorer/specialty_anesthesia_std": 0.0904, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.4092, | |
| "_scorer/specialty_cards_std": 0.0608, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.6483, | |
| "_scorer/specialty_derm_std": 0.0818, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.4549, | |
| "_scorer/specialty_emergency_std": 0.1171, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5268, | |
| "_scorer/specialty_endo_std": 0.0814, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.7955, | |
| "_scorer/specialty_genetics_std": 0.1101, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.5019, | |
| "_scorer/specialty_heme/onc_std": 0.0834, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.6713, | |
| "_scorer/specialty_medicine_std": 0.1017, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.4311, | |
| "_scorer/specialty_nephro_std": 0.0829, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.5592, | |
| "_scorer/specialty_neuro_std": 0.0608, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.3242, | |
| "_scorer/specialty_neurosurg_std": 0.1113, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.4791, | |
| "_scorer/specialty_obgyn_std": 0.0631, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.3167, | |
| "_scorer/specialty_ophtho_std": 0.0948, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.3507, | |
| "_scorer/specialty_ortho_std": 0.0905, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.3541, | |
| "_scorer/specialty_other_std": 0.0877, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.6667, | |
| "_scorer/specialty_path_std": 0.2746, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.5363, | |
| "_scorer/specialty_peds_std": 0.0828, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.563, | |
| "_scorer/specialty_primary_std": 0.1582, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.5115, | |
| "_scorer/specialty_psychiatry_std": 0.0696, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.5848, | |
| "_scorer/specialty_pulm_std": 0.1163, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.3787, | |
| "_scorer/specialty_rads_std": 0.1382, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.6964, | |
| "_scorer/specialty_rheum_std": 0.1529, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.498, | |
| "_scorer/specialty_surgery_std": 0.1205, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.6175, | |
| "_scorer/specialty_uro_std": 0.1159, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.5803, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0224, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.4768, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0513, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.4137, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0322, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.4429, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4387 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "physician_baseline_solver" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T19-33-04-00-00_healthbench-professional-consult_dgWnf9y2uVjcWoDcyTm3Mm.eval", | |
| "task": "healthbench_professional_consult", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai-api/plamo/plamo-3.0-prime", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 236, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 94, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T19:33:04+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1888, | |
| "total_samples": 1888, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.1536, | |
| "_scorer/std": 0.0318, | |
| "_scorer/criteria_met_rate": 0.4021, | |
| "_scorer/use_case_consult_score": 0.3313, | |
| "_scorer/use_case_consult_std": 0.027, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/type_good_faith_score": 0.3876, | |
| "_scorer/type_good_faith_std": 0.0362, | |
| "_scorer/type_good_faith_n_samples": 143.0, | |
| "_scorer/type_red_teaming_score": 0.2446, | |
| "_scorer/type_red_teaming_std": 0.0421, | |
| "_scorer/type_red_teaming_n_samples": 93.0, | |
| "_scorer/difficulty_typical_score": 0.4979, | |
| "_scorer/difficulty_typical_std": 0.0436, | |
| "_scorer/difficulty_typical_n_samples": 84.0, | |
| "_scorer/difficulty_difficult_score": 0.2391, | |
| "_scorer/difficulty_difficult_std": 0.0315, | |
| "_scorer/difficulty_difficult_n_samples": 152.0, | |
| "_scorer/specialty_ent_score": 0.6, | |
| "_scorer/specialty_ent_std": 0.2166, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.0, | |
| "_scorer/specialty_gi_std": 0.0, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.0833, | |
| "_scorer/specialty_id_std": 0.0759, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.3553, | |
| "_scorer/specialty_anesthesia_std": 0.1369, | |
| "_scorer/specialty_anesthesia_n_samples": 10.0, | |
| "_scorer/specialty_cards_score": 0.329, | |
| "_scorer/specialty_cards_std": 0.0898, | |
| "_scorer/specialty_cards_n_samples": 23.0, | |
| "_scorer/specialty_derm_score": 0.339, | |
| "_scorer/specialty_derm_std": 0.1009, | |
| "_scorer/specialty_derm_n_samples": 14.0, | |
| "_scorer/specialty_emergency_score": 0.5588, | |
| "_scorer/specialty_emergency_std": 0.1589, | |
| "_scorer/specialty_emergency_n_samples": 8.0, | |
| "_scorer/specialty_endo_score": 0.4439, | |
| "_scorer/specialty_endo_std": 0.1273, | |
| "_scorer/specialty_endo_n_samples": 9.0, | |
| "_scorer/specialty_genetics_score": 0.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.2, | |
| "_scorer/specialty_heme/onc_std": 0.1241, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.1667, | |
| "_scorer/specialty_medicine_std": 0.1488, | |
| "_scorer/specialty_medicine_n_samples": 6.0, | |
| "_scorer/specialty_nephro_score": 0.509, | |
| "_scorer/specialty_nephro_std": 0.1374, | |
| "_scorer/specialty_nephro_n_samples": 9.0, | |
| "_scorer/specialty_neuro_score": 0.4353, | |
| "_scorer/specialty_neuro_std": 0.1036, | |
| "_scorer/specialty_neuro_n_samples": 15.0, | |
| "_scorer/specialty_neurosurg_score": 0.6294, | |
| "_scorer/specialty_neurosurg_std": 0.1542, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.3456, | |
| "_scorer/specialty_obgyn_std": 0.0907, | |
| "_scorer/specialty_obgyn_n_samples": 18.0, | |
| "_scorer/specialty_ophtho_score": 0.1511, | |
| "_scorer/specialty_ophtho_std": 0.1075, | |
| "_scorer/specialty_ophtho_n_samples": 9.0, | |
| "_scorer/specialty_ortho_score": 0.3846, | |
| "_scorer/specialty_ortho_std": 0.1355, | |
| "_scorer/specialty_ortho_n_samples": 13.0, | |
| "_scorer/specialty_other_score": 0.1667, | |
| "_scorer/specialty_other_std": 0.1536, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_path_score": 0.3333, | |
| "_scorer/specialty_path_std": 0.2622, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.1807, | |
| "_scorer/specialty_peds_std": 0.0936, | |
| "_scorer/specialty_peds_n_samples": 14.0, | |
| "_scorer/specialty_primary_score": 0.6176, | |
| "_scorer/specialty_primary_std": 0.2129, | |
| "_scorer/specialty_primary_n_samples": 4.0, | |
| "_scorer/specialty_psychiatry_score": 0.5006, | |
| "_scorer/specialty_psychiatry_std": 0.1437, | |
| "_scorer/specialty_psychiatry_n_samples": 7.0, | |
| "_scorer/specialty_pulm_score": 0.1562, | |
| "_scorer/specialty_pulm_std": 0.0936, | |
| "_scorer/specialty_pulm_n_samples": 6.0, | |
| "_scorer/specialty_rads_score": 0.2143, | |
| "_scorer/specialty_rads_std": 0.1346, | |
| "_scorer/specialty_rads_n_samples": 7.0, | |
| "_scorer/specialty_rheum_score": 0.3762, | |
| "_scorer/specialty_rheum_std": 0.1664, | |
| "_scorer/specialty_rheum_n_samples": 5.0, | |
| "_scorer/specialty_surgery_score": 0.35, | |
| "_scorer/specialty_surgery_std": 0.144, | |
| "_scorer/specialty_surgery_n_samples": 10.0, | |
| "_scorer/specialty_uro_score": 0.5388, | |
| "_scorer/specialty_uro_std": 0.1826, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.4979, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0415, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 84.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.2305, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0522, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2446, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0435, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, | |
| "healthbench_professional_score/mean": 0.2184, | |
| "healthbench_professional_score_length_adjusted/mean": 0.1536 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T19-33-35-00-00_healthbench-professional-writing_eAmnviGLop8QfjxGLMp9Dz.eval", | |
| "task": "healthbench_professional_writing", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai-api/plamo/plamo-3.0-prime", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 142, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 56, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T19:33:35+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1136, | |
| "total_samples": 1136, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0, | |
| "_scorer/std": 0.0147, | |
| "_scorer/criteria_met_rate": 0.3924, | |
| "_scorer/use_case_writing_score": 0.1977, | |
| "_scorer/use_case_writing_std": 0.0287, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_red_teaming_score": 0.0495, | |
| "_scorer/type_red_teaming_std": 0.0226, | |
| "_scorer/type_red_teaming_n_samples": 91.0, | |
| "_scorer/type_good_faith_score": 0.4623, | |
| "_scorer/type_good_faith_std": 0.0573, | |
| "_scorer/type_good_faith_n_samples": 51.0, | |
| "_scorer/difficulty_difficult_score": 0.048, | |
| "_scorer/difficulty_difficult_std": 0.0217, | |
| "_scorer/difficulty_difficult_n_samples": 96.0, | |
| "_scorer/difficulty_typical_score": 0.5101, | |
| "_scorer/difficulty_typical_std": 0.0578, | |
| "_scorer/difficulty_typical_n_samples": 46.0, | |
| "_scorer/specialty_ent_score": 0.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 2.0, | |
| "_scorer/specialty_gi_score": 0.0, | |
| "_scorer/specialty_gi_std": 0.0, | |
| "_scorer/specialty_gi_n_samples": 5.0, | |
| "_scorer/specialty_id_score": 0.0764, | |
| "_scorer/specialty_id_std": 0.0581, | |
| "_scorer/specialty_id_n_samples": 8.0, | |
| "_scorer/specialty_anesthesia_score": 0.2545, | |
| "_scorer/specialty_anesthesia_std": 0.1274, | |
| "_scorer/specialty_anesthesia_n_samples": 4.0, | |
| "_scorer/specialty_cards_score": 0.524, | |
| "_scorer/specialty_cards_std": 0.1529, | |
| "_scorer/specialty_cards_n_samples": 6.0, | |
| "_scorer/specialty_derm_score": 0.0929, | |
| "_scorer/specialty_derm_std": 0.0846, | |
| "_scorer/specialty_derm_n_samples": 7.0, | |
| "_scorer/specialty_emergency_score": 0.1429, | |
| "_scorer/specialty_emergency_std": 0.1351, | |
| "_scorer/specialty_emergency_n_samples": 7.0, | |
| "_scorer/specialty_endo_score": 0.1804, | |
| "_scorer/specialty_endo_std": 0.1028, | |
| "_scorer/specialty_endo_n_samples": 5.0, | |
| "_scorer/specialty_genetics_score": 1.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.263, | |
| "_scorer/specialty_heme/onc_std": 0.1242, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.0, | |
| "_scorer/specialty_medicine_std": 0.0, | |
| "_scorer/specialty_medicine_n_samples": 3.0, | |
| "_scorer/specialty_nephro_score": 0.2371, | |
| "_scorer/specialty_nephro_std": 0.1017, | |
| "_scorer/specialty_nephro_n_samples": 6.0, | |
| "_scorer/specialty_neuro_score": 0.1649, | |
| "_scorer/specialty_neuro_std": 0.1124, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.25, | |
| "_scorer/specialty_neurosurg_std": 0.2179, | |
| "_scorer/specialty_neurosurg_n_samples": 4.0, | |
| "_scorer/specialty_obgyn_score": 0.25, | |
| "_scorer/specialty_obgyn_std": 0.1256, | |
| "_scorer/specialty_obgyn_n_samples": 12.0, | |
| "_scorer/specialty_ophtho_score": 0.5333, | |
| "_scorer/specialty_ophtho_std": 0.1985, | |
| "_scorer/specialty_ophtho_n_samples": 5.0, | |
| "_scorer/specialty_ortho_score": 0.0, | |
| "_scorer/specialty_ortho_std": 0.0, | |
| "_scorer/specialty_ortho_n_samples": 4.0, | |
| "_scorer/specialty_other_score": 0.1667, | |
| "_scorer/specialty_other_std": 0.1175, | |
| "_scorer/specialty_other_n_samples": 8.0, | |
| "_scorer/specialty_peds_score": 0.25, | |
| "_scorer/specialty_peds_std": 0.1238, | |
| "_scorer/specialty_peds_n_samples": 8.0, | |
| "_scorer/specialty_primary_score": 0.3333, | |
| "_scorer/specialty_primary_std": 0.2731, | |
| "_scorer/specialty_primary_n_samples": 3.0, | |
| "_scorer/specialty_psychiatry_score": 0.1846, | |
| "_scorer/specialty_psychiatry_std": 0.099, | |
| "_scorer/specialty_psychiatry_n_samples": 12.0, | |
| "_scorer/specialty_pulm_score": 0.0, | |
| "_scorer/specialty_pulm_std": 0.0, | |
| "_scorer/specialty_pulm_n_samples": 2.0, | |
| "_scorer/specialty_rads_score": 0.0, | |
| "_scorer/specialty_rads_std": 0.0, | |
| "_scorer/specialty_rads_n_samples": 1.0, | |
| "_scorer/specialty_rheum_score": 0.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 2.0, | |
| "_scorer/specialty_surgery_score": 0.25, | |
| "_scorer/specialty_surgery_std": 0.2172, | |
| "_scorer/specialty_surgery_n_samples": 4.0, | |
| "_scorer/specialty_uro_score": 0.0, | |
| "_scorer/specialty_uro_std": 0.0, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.0495, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0225, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.0222, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0203, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.5101, | |
| "_scorer/source_slice_good_faith_typical_std": 0.059, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 46.0, | |
| "healthbench_professional_score/mean": -0.0276, | |
| "healthbench_professional_score_length_adjusted/mean": -0.0431 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T19-33-52-00-00_healthbench-professional-research_WF5VpCWo3K285bwUiVdCUm.eval", | |
| "task": "healthbench_professional_research", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai-api/plamo/plamo-3.0-prime", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 147, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 58, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T19:33:52+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1176, | |
| "total_samples": 1176, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2858, | |
| "_scorer/std": 0.036, | |
| "_scorer/criteria_met_rate": 0.4722, | |
| "_scorer/use_case_research_score": 0.4886, | |
| "_scorer/use_case_research_std": 0.0344, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/type_red_teaming_score": 0.7143, | |
| "_scorer/type_red_teaming_std": 0.1786, | |
| "_scorer/type_red_teaming_n_samples": 7.0, | |
| "_scorer/type_good_faith_score": 0.4773, | |
| "_scorer/type_good_faith_std": 0.0344, | |
| "_scorer/type_good_faith_n_samples": 140.0, | |
| "_scorer/difficulty_difficult_score": 0.4762, | |
| "_scorer/difficulty_difficult_std": 0.1095, | |
| "_scorer/difficulty_difficult_n_samples": 21.0, | |
| "_scorer/difficulty_typical_score": 0.4907, | |
| "_scorer/difficulty_typical_std": 0.0348, | |
| "_scorer/difficulty_typical_n_samples": 126.0, | |
| "_scorer/specialty_ent_score": 1.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 1.0, | |
| "_scorer/specialty_gi_score": 0.481, | |
| "_scorer/specialty_gi_std": 0.1484, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.7718, | |
| "_scorer/specialty_id_std": 0.0991, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_anesthesia_score": 0.5089, | |
| "_scorer/specialty_anesthesia_std": 0.1538, | |
| "_scorer/specialty_anesthesia_n_samples": 8.0, | |
| "_scorer/specialty_cards_score": 0.4794, | |
| "_scorer/specialty_cards_std": 0.0889, | |
| "_scorer/specialty_cards_n_samples": 14.0, | |
| "_scorer/specialty_derm_score": 0.5, | |
| "_scorer/specialty_derm_std": 0.2454, | |
| "_scorer/specialty_derm_n_samples": 4.0, | |
| "_scorer/specialty_emergency_score": 1.0, | |
| "_scorer/specialty_emergency_std": 0.0, | |
| "_scorer/specialty_emergency_n_samples": 1.0, | |
| "_scorer/specialty_endo_score": 0.8867, | |
| "_scorer/specialty_endo_std": 0.1064, | |
| "_scorer/specialty_endo_n_samples": 6.0, | |
| "_scorer/specialty_genetics_score": 1.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 2.0, | |
| "_scorer/specialty_heme/onc_score": 0.2597, | |
| "_scorer/specialty_heme/onc_std": 0.1351, | |
| "_scorer/specialty_heme/onc_n_samples": 7.0, | |
| "_scorer/specialty_medicine_score": 0.5043, | |
| "_scorer/specialty_medicine_std": 0.1589, | |
| "_scorer/specialty_medicine_n_samples": 8.0, | |
| "_scorer/specialty_nephro_score": 0.5323, | |
| "_scorer/specialty_nephro_std": 0.1498, | |
| "_scorer/specialty_nephro_n_samples": 7.0, | |
| "_scorer/specialty_neuro_score": 0.4841, | |
| "_scorer/specialty_neuro_std": 0.0912, | |
| "_scorer/specialty_neuro_n_samples": 19.0, | |
| "_scorer/specialty_neurosurg_score": 0.0, | |
| "_scorer/specialty_neurosurg_std": 0.0, | |
| "_scorer/specialty_neurosurg_n_samples": 1.0, | |
| "_scorer/specialty_obgyn_score": 0.4333, | |
| "_scorer/specialty_obgyn_std": 0.144, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 0.75, | |
| "_scorer/specialty_ophtho_std": 0.2117, | |
| "_scorer/specialty_ophtho_n_samples": 4.0, | |
| "_scorer/specialty_ortho_score": 0.4238, | |
| "_scorer/specialty_ortho_std": 0.1561, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.3538, | |
| "_scorer/specialty_other_std": 0.115, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_peds_score": 0.507, | |
| "_scorer/specialty_peds_std": 0.1691, | |
| "_scorer/specialty_peds_n_samples": 6.0, | |
| "_scorer/specialty_psychiatry_score": 0.1877, | |
| "_scorer/specialty_psychiatry_std": 0.1041, | |
| "_scorer/specialty_psychiatry_n_samples": 10.0, | |
| "_scorer/specialty_pulm_score": 0.4224, | |
| "_scorer/specialty_pulm_std": 0.2232, | |
| "_scorer/specialty_pulm_n_samples": 4.0, | |
| "_scorer/specialty_rheum_score": 1.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 1.0, | |
| "_scorer/specialty_surgery_score": 0.3222, | |
| "_scorer/specialty_surgery_std": 0.1326, | |
| "_scorer/specialty_surgery_n_samples": 3.0, | |
| "_scorer/specialty_uro_score": 0.3894, | |
| "_scorer/specialty_uro_std": 0.1587, | |
| "_scorer/specialty_uro_n_samples": 5.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.7143, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.172, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.4907, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0366, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 126.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3571, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.1272, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, | |
| "healthbench_professional_score/mean": 0.4183, | |
| "healthbench_professional_score_length_adjusted/mean": 0.2858 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T19-34-11-00-00_healthbench-professional-red-teaming_4db2cTWDv6VCVoFSFqWpmW.eval", | |
| "task": "healthbench_professional_red_teaming", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai-api/plamo/plamo-3.0-prime", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 191, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 76, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T19:34:11+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1528, | |
| "total_samples": 1528, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0, | |
| "_scorer/std": 0.0005, | |
| "_scorer/criteria_met_rate": 0.3504, | |
| "_scorer/use_case_consult_score": 0.2446, | |
| "_scorer/use_case_consult_std": 0.0417, | |
| "_scorer/use_case_consult_n_samples": 93.0, | |
| "_scorer/use_case_research_score": 0.7143, | |
| "_scorer/use_case_research_std": 0.1669, | |
| "_scorer/use_case_research_n_samples": 7.0, | |
| "_scorer/use_case_writing_score": 0.0495, | |
| "_scorer/use_case_writing_std": 0.0224, | |
| "_scorer/use_case_writing_n_samples": 91.0, | |
| "_scorer/type_red_teaming_score": 0.1688, | |
| "_scorer/type_red_teaming_std": 0.0263, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_difficult_score": 0.1688, | |
| "_scorer/difficulty_difficult_std": 0.0256, | |
| "_scorer/difficulty_difficult_n_samples": 191.0, | |
| "_scorer/specialty_ent_score": 0.5, | |
| "_scorer/specialty_ent_std": 0.2499, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.0, | |
| "_scorer/specialty_gi_std": 0.0, | |
| "_scorer/specialty_gi_n_samples": 8.0, | |
| "_scorer/specialty_id_score": 0.1, | |
| "_scorer/specialty_id_std": 0.0608, | |
| "_scorer/specialty_id_n_samples": 10.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 1.0, | |
| "_scorer/specialty_anesthesia_score": 0.1845, | |
| "_scorer/specialty_anesthesia_std": 0.1331, | |
| "_scorer/specialty_anesthesia_n_samples": 7.0, | |
| "_scorer/specialty_cards_score": 0.423, | |
| "_scorer/specialty_cards_std": 0.1408, | |
| "_scorer/specialty_cards_n_samples": 11.0, | |
| "_scorer/specialty_derm_score": 0.0481, | |
| "_scorer/specialty_derm_std": 0.045, | |
| "_scorer/specialty_derm_n_samples": 11.0, | |
| "_scorer/specialty_emergency_score": 0.3636, | |
| "_scorer/specialty_emergency_std": 0.1472, | |
| "_scorer/specialty_emergency_n_samples": 11.0, | |
| "_scorer/specialty_endo_score": 0.3429, | |
| "_scorer/specialty_endo_std": 0.1422, | |
| "_scorer/specialty_endo_n_samples": 10.0, | |
| "_scorer/specialty_genetics_score": 0.6667, | |
| "_scorer/specialty_genetics_std": 0.2708, | |
| "_scorer/specialty_genetics_n_samples": 3.0, | |
| "_scorer/specialty_heme/onc_score": 0.0, | |
| "_scorer/specialty_heme/onc_std": 0.0, | |
| "_scorer/specialty_heme/onc_n_samples": 11.0, | |
| "_scorer/specialty_medicine_score": 0.1429, | |
| "_scorer/specialty_medicine_std": 0.1296, | |
| "_scorer/specialty_medicine_n_samples": 7.0, | |
| "_scorer/specialty_nephro_score": 0.25, | |
| "_scorer/specialty_nephro_std": 0.2105, | |
| "_scorer/specialty_nephro_n_samples": 4.0, | |
| "_scorer/specialty_neuro_score": 0.2267, | |
| "_scorer/specialty_neuro_std": 0.1397, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.2961, | |
| "_scorer/specialty_neurosurg_std": 0.1449, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.1, | |
| "_scorer/specialty_obgyn_std": 0.0942, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 0.0, | |
| "_scorer/specialty_ophtho_std": 0.0, | |
| "_scorer/specialty_ophtho_n_samples": 3.0, | |
| "_scorer/specialty_ortho_score": 0.1429, | |
| "_scorer/specialty_ortho_std": 0.1272, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.0, | |
| "_scorer/specialty_other_std": 0.0, | |
| "_scorer/specialty_other_n_samples": 9.0, | |
| "_scorer/specialty_path_score": 1.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 1.0, | |
| "_scorer/specialty_peds_score": 0.0529, | |
| "_scorer/specialty_peds_std": 0.0497, | |
| "_scorer/specialty_peds_n_samples": 10.0, | |
| "_scorer/specialty_primary_score": 0.4, | |
| "_scorer/specialty_primary_std": 0.2181, | |
| "_scorer/specialty_primary_n_samples": 5.0, | |
| "_scorer/specialty_psychiatry_score": 0.1111, | |
| "_scorer/specialty_psychiatry_std": 0.1049, | |
| "_scorer/specialty_psychiatry_n_samples": 9.0, | |
| "_scorer/specialty_pulm_score": 0.0, | |
| "_scorer/specialty_pulm_std": 0.0, | |
| "_scorer/specialty_pulm_n_samples": 5.0, | |
| "_scorer/specialty_rads_score": 0.0, | |
| "_scorer/specialty_rads_std": 0.0, | |
| "_scorer/specialty_rads_n_samples": 5.0, | |
| "_scorer/specialty_rheum_score": 0.25, | |
| "_scorer/specialty_rheum_std": 0.22, | |
| "_scorer/specialty_rheum_n_samples": 4.0, | |
| "_scorer/specialty_surgery_score": 0.0, | |
| "_scorer/specialty_surgery_std": 0.0, | |
| "_scorer/specialty_surgery_n_samples": 7.0, | |
| "_scorer/specialty_uro_score": 0.0, | |
| "_scorer/specialty_uro_std": 0.0, | |
| "_scorer/specialty_uro_n_samples": 3.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.1688, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0252, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": -0.0986, | |
| "healthbench_professional_score_length_adjusted/mean": -0.1182 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T19-34-35-00-00_healthbench-professional-physician-baseline_S4NuFZgLXeK7RAgHpRKqTy.eval", | |
| "task": "healthbench_professional_physician_baseline", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openai-api/plamo/plamo-3.0-prime", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T19:34:35+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4387, | |
| "_scorer/std": 0.0237, | |
| "_scorer/criteria_met_rate": 0.5044, | |
| "_scorer/use_case_consult_score": 0.4843, | |
| "_scorer/use_case_consult_std": 0.0294, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.59, | |
| "_scorer/use_case_research_std": 0.0301, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.449, | |
| "_scorer/use_case_writing_std": 0.0351, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.5561, | |
| "_scorer/type_good_faith_std": 0.0208, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.4137, | |
| "_scorer/type_red_teaming_std": 0.0335, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.5803, | |
| "_scorer/difficulty_typical_std": 0.0224, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.432, | |
| "_scorer/difficulty_difficult_std": 0.0284, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.4857, | |
| "_scorer/specialty_ent_std": 0.1752, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.5427, | |
| "_scorer/specialty_gi_std": 0.0948, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.5113, | |
| "_scorer/specialty_id_std": 0.072, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.6229, | |
| "_scorer/specialty_anesthesia_std": 0.0868, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.4092, | |
| "_scorer/specialty_cards_std": 0.0613, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.6483, | |
| "_scorer/specialty_derm_std": 0.0795, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.4549, | |
| "_scorer/specialty_emergency_std": 0.1181, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5268, | |
| "_scorer/specialty_endo_std": 0.0848, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.7955, | |
| "_scorer/specialty_genetics_std": 0.1043, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.5019, | |
| "_scorer/specialty_heme/onc_std": 0.0828, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.6713, | |
| "_scorer/specialty_medicine_std": 0.0961, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.4311, | |
| "_scorer/specialty_nephro_std": 0.0806, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.5592, | |
| "_scorer/specialty_neuro_std": 0.059, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.3242, | |
| "_scorer/specialty_neurosurg_std": 0.1149, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.4791, | |
| "_scorer/specialty_obgyn_std": 0.0644, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.3167, | |
| "_scorer/specialty_ophtho_std": 0.1, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.3507, | |
| "_scorer/specialty_ortho_std": 0.0866, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.3541, | |
| "_scorer/specialty_other_std": 0.0879, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.6667, | |
| "_scorer/specialty_path_std": 0.2675, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.5363, | |
| "_scorer/specialty_peds_std": 0.0829, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.563, | |
| "_scorer/specialty_primary_std": 0.1577, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.5115, | |
| "_scorer/specialty_psychiatry_std": 0.074, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.5848, | |
| "_scorer/specialty_pulm_std": 0.1155, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.3787, | |
| "_scorer/specialty_rads_std": 0.1435, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.6964, | |
| "_scorer/specialty_rheum_std": 0.1477, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.498, | |
| "_scorer/specialty_surgery_std": 0.1199, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.6175, | |
| "_scorer/specialty_uro_std": 0.1167, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.5803, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0225, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.4768, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0528, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.4137, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0321, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.4429, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4387 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "physician_baseline_solver" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T19-59-01-00-00_healthbench-professional-writing_933975rUF6cdWKJECk44bJ.eval", | |
| "task": "healthbench_professional_writing", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 142, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 56, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T19:59:01+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1136, | |
| "total_samples": 1136, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0495, | |
| "_scorer/std": 0.0457, | |
| "_scorer/criteria_met_rate": 0.5576, | |
| "_scorer/use_case_writing_score": 0.346, | |
| "_scorer/use_case_writing_std": 0.0362, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_red_teaming_score": 0.1183, | |
| "_scorer/type_red_teaming_std": 0.032, | |
| "_scorer/type_red_teaming_n_samples": 91.0, | |
| "_scorer/type_good_faith_score": 0.7522, | |
| "_scorer/type_good_faith_std": 0.0469, | |
| "_scorer/type_good_faith_n_samples": 51.0, | |
| "_scorer/difficulty_difficult_score": 0.1486, | |
| "_scorer/difficulty_difficult_std": 0.0312, | |
| "_scorer/difficulty_difficult_n_samples": 96.0, | |
| "_scorer/difficulty_typical_score": 0.7579, | |
| "_scorer/difficulty_typical_std": 0.046, | |
| "_scorer/difficulty_typical_n_samples": 46.0, | |
| "_scorer/specialty_ent_score": 0.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 2.0, | |
| "_scorer/specialty_gi_score": 0.2, | |
| "_scorer/specialty_gi_std": 0.1788, | |
| "_scorer/specialty_gi_n_samples": 5.0, | |
| "_scorer/specialty_id_score": 0.275, | |
| "_scorer/specialty_id_std": 0.1219, | |
| "_scorer/specialty_id_n_samples": 8.0, | |
| "_scorer/specialty_anesthesia_score": 0.4318, | |
| "_scorer/specialty_anesthesia_std": 0.2228, | |
| "_scorer/specialty_anesthesia_n_samples": 4.0, | |
| "_scorer/specialty_cards_score": 0.5694, | |
| "_scorer/specialty_cards_std": 0.1709, | |
| "_scorer/specialty_cards_n_samples": 6.0, | |
| "_scorer/specialty_derm_score": 0.1429, | |
| "_scorer/specialty_derm_std": 0.1317, | |
| "_scorer/specialty_derm_n_samples": 7.0, | |
| "_scorer/specialty_emergency_score": 0.1429, | |
| "_scorer/specialty_emergency_std": 0.133, | |
| "_scorer/specialty_emergency_n_samples": 7.0, | |
| "_scorer/specialty_endo_score": 0.3474, | |
| "_scorer/specialty_endo_std": 0.187, | |
| "_scorer/specialty_endo_n_samples": 5.0, | |
| "_scorer/specialty_genetics_score": 0.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.277, | |
| "_scorer/specialty_heme/onc_std": 0.13, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.3333, | |
| "_scorer/specialty_medicine_std": 0.2718, | |
| "_scorer/specialty_medicine_n_samples": 3.0, | |
| "_scorer/specialty_nephro_score": 0.3359, | |
| "_scorer/specialty_nephro_std": 0.1558, | |
| "_scorer/specialty_nephro_n_samples": 6.0, | |
| "_scorer/specialty_neuro_score": 0.4409, | |
| "_scorer/specialty_neuro_std": 0.1469, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.375, | |
| "_scorer/specialty_neurosurg_std": 0.2108, | |
| "_scorer/specialty_neurosurg_n_samples": 4.0, | |
| "_scorer/specialty_obgyn_score": 0.5491, | |
| "_scorer/specialty_obgyn_std": 0.1146, | |
| "_scorer/specialty_obgyn_n_samples": 12.0, | |
| "_scorer/specialty_ophtho_score": 0.475, | |
| "_scorer/specialty_ophtho_std": 0.2042, | |
| "_scorer/specialty_ophtho_n_samples": 5.0, | |
| "_scorer/specialty_ortho_score": 0.25, | |
| "_scorer/specialty_ortho_std": 0.2124, | |
| "_scorer/specialty_ortho_n_samples": 4.0, | |
| "_scorer/specialty_other_score": 0.2917, | |
| "_scorer/specialty_other_std": 0.1461, | |
| "_scorer/specialty_other_n_samples": 8.0, | |
| "_scorer/specialty_peds_score": 0.625, | |
| "_scorer/specialty_peds_std": 0.1446, | |
| "_scorer/specialty_peds_n_samples": 8.0, | |
| "_scorer/specialty_primary_score": 0.3333, | |
| "_scorer/specialty_primary_std": 0.2746, | |
| "_scorer/specialty_primary_n_samples": 3.0, | |
| "_scorer/specialty_psychiatry_score": 0.5028, | |
| "_scorer/specialty_psychiatry_std": 0.1284, | |
| "_scorer/specialty_psychiatry_n_samples": 12.0, | |
| "_scorer/specialty_pulm_score": 0.0, | |
| "_scorer/specialty_pulm_std": 0.0, | |
| "_scorer/specialty_pulm_n_samples": 2.0, | |
| "_scorer/specialty_rads_score": 0.4667, | |
| "_scorer/specialty_rads_std": 0.0, | |
| "_scorer/specialty_rads_n_samples": 1.0, | |
| "_scorer/specialty_rheum_score": 0.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 2.0, | |
| "_scorer/specialty_surgery_score": 0.25, | |
| "_scorer/specialty_surgery_std": 0.2225, | |
| "_scorer/specialty_surgery_n_samples": 4.0, | |
| "_scorer/specialty_uro_score": 0.0, | |
| "_scorer/specialty_uro_std": 0.0, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.1183, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0305, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.7, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.1781, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7579, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0467, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 46.0, | |
| "healthbench_professional_score/mean": 0.0954, | |
| "healthbench_professional_score_length_adjusted/mean": 0.0495 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-5.4", | |
| "openrouter/deepseek/deepseek-v4-pro" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-5.4": 3316215, | |
| "openrouter/deepseek/deepseek-v4-pro": 1479207 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T20-07-30-00-00_healthbench-professional-research_7gKGNrErBwsumd7oKKUcuN.eval", | |
| "task": "healthbench_professional_research", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 147, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 58, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T20:07:30+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1176, | |
| "total_samples": 1176, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.5291, | |
| "_scorer/std": 0.0307, | |
| "_scorer/criteria_met_rate": 0.6757, | |
| "_scorer/use_case_research_score": 0.6937, | |
| "_scorer/use_case_research_std": 0.0317, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/type_red_teaming_score": 0.8571, | |
| "_scorer/type_red_teaming_std": 0.1272, | |
| "_scorer/type_red_teaming_n_samples": 7.0, | |
| "_scorer/type_good_faith_score": 0.6855, | |
| "_scorer/type_good_faith_std": 0.0319, | |
| "_scorer/type_good_faith_n_samples": 140.0, | |
| "_scorer/difficulty_difficult_score": 0.5952, | |
| "_scorer/difficulty_difficult_std": 0.1028, | |
| "_scorer/difficulty_difficult_n_samples": 21.0, | |
| "_scorer/difficulty_typical_score": 0.7101, | |
| "_scorer/difficulty_typical_std": 0.0304, | |
| "_scorer/difficulty_typical_n_samples": 126.0, | |
| "_scorer/specialty_ent_score": 1.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 1.0, | |
| "_scorer/specialty_gi_score": 0.6869, | |
| "_scorer/specialty_gi_std": 0.1124, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.7302, | |
| "_scorer/specialty_id_std": 0.0839, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_anesthesia_score": 0.8214, | |
| "_scorer/specialty_anesthesia_std": 0.0811, | |
| "_scorer/specialty_anesthesia_n_samples": 8.0, | |
| "_scorer/specialty_cards_score": 0.7352, | |
| "_scorer/specialty_cards_std": 0.0923, | |
| "_scorer/specialty_cards_n_samples": 14.0, | |
| "_scorer/specialty_derm_score": 1.0, | |
| "_scorer/specialty_derm_std": 0.0, | |
| "_scorer/specialty_derm_n_samples": 4.0, | |
| "_scorer/specialty_emergency_score": 0.0, | |
| "_scorer/specialty_emergency_std": 0.0, | |
| "_scorer/specialty_emergency_n_samples": 1.0, | |
| "_scorer/specialty_endo_score": 0.8567, | |
| "_scorer/specialty_endo_std": 0.0856, | |
| "_scorer/specialty_endo_n_samples": 6.0, | |
| "_scorer/specialty_genetics_score": 1.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 2.0, | |
| "_scorer/specialty_heme/onc_score": 0.6926, | |
| "_scorer/specialty_heme/onc_std": 0.1316, | |
| "_scorer/specialty_heme/onc_n_samples": 7.0, | |
| "_scorer/specialty_medicine_score": 0.427, | |
| "_scorer/specialty_medicine_std": 0.155, | |
| "_scorer/specialty_medicine_n_samples": 8.0, | |
| "_scorer/specialty_nephro_score": 0.8538, | |
| "_scorer/specialty_nephro_std": 0.0906, | |
| "_scorer/specialty_nephro_n_samples": 7.0, | |
| "_scorer/specialty_neuro_score": 0.672, | |
| "_scorer/specialty_neuro_std": 0.0899, | |
| "_scorer/specialty_neuro_n_samples": 19.0, | |
| "_scorer/specialty_neurosurg_score": 0.0667, | |
| "_scorer/specialty_neurosurg_std": 0.0, | |
| "_scorer/specialty_neurosurg_n_samples": 1.0, | |
| "_scorer/specialty_obgyn_score": 0.4816, | |
| "_scorer/specialty_obgyn_std": 0.1137, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 1.0, | |
| "_scorer/specialty_ophtho_std": 0.0, | |
| "_scorer/specialty_ophtho_n_samples": 4.0, | |
| "_scorer/specialty_ortho_score": 0.2988, | |
| "_scorer/specialty_ortho_std": 0.1309, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.6736, | |
| "_scorer/specialty_other_std": 0.1413, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_peds_score": 0.7143, | |
| "_scorer/specialty_peds_std": 0.1645, | |
| "_scorer/specialty_peds_n_samples": 6.0, | |
| "_scorer/specialty_psychiatry_score": 0.725, | |
| "_scorer/specialty_psychiatry_std": 0.1327, | |
| "_scorer/specialty_psychiatry_n_samples": 10.0, | |
| "_scorer/specialty_pulm_score": 0.6026, | |
| "_scorer/specialty_pulm_std": 0.1386, | |
| "_scorer/specialty_pulm_n_samples": 4.0, | |
| "_scorer/specialty_rheum_score": 1.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 1.0, | |
| "_scorer/specialty_surgery_score": 1.0, | |
| "_scorer/specialty_surgery_std": 0.0, | |
| "_scorer/specialty_surgery_n_samples": 3.0, | |
| "_scorer/specialty_uro_score": 0.7609, | |
| "_scorer/specialty_uro_std": 0.1327, | |
| "_scorer/specialty_uro_n_samples": 5.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.8571, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.1309, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7101, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0315, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 126.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.4643, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.1296, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, | |
| "healthbench_professional_score/mean": 0.6368, | |
| "healthbench_professional_score_length_adjusted/mean": 0.5291 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-5.4", | |
| "openrouter/deepseek/deepseek-v4-pro" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-5.4": 4316530, | |
| "openrouter/deepseek/deepseek-v4-pro": 2021029 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T20-18-24-00-00_healthbench-professional-red-teaming_HmwUcZnfLNqmUkyZkHDTdJ.eval", | |
| "task": "healthbench_professional_red_teaming", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 191, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 76, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T20:18:24+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1528, | |
| "total_samples": 1528, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0, | |
| "_scorer/std": 0.0072, | |
| "_scorer/criteria_met_rate": 0.4732, | |
| "_scorer/use_case_consult_score": 0.3379, | |
| "_scorer/use_case_consult_std": 0.0446, | |
| "_scorer/use_case_consult_n_samples": 93.0, | |
| "_scorer/use_case_research_score": 0.8571, | |
| "_scorer/use_case_research_std": 0.1282, | |
| "_scorer/use_case_research_n_samples": 7.0, | |
| "_scorer/use_case_writing_score": 0.1183, | |
| "_scorer/use_case_writing_std": 0.0301, | |
| "_scorer/use_case_writing_n_samples": 91.0, | |
| "_scorer/type_red_teaming_score": 0.2523, | |
| "_scorer/type_red_teaming_std": 0.0296, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_difficult_score": 0.2523, | |
| "_scorer/difficulty_difficult_std": 0.0302, | |
| "_scorer/difficulty_difficult_n_samples": 191.0, | |
| "_scorer/specialty_ent_score": 0.5, | |
| "_scorer/specialty_ent_std": 0.2452, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.0, | |
| "_scorer/specialty_gi_std": 0.0, | |
| "_scorer/specialty_gi_n_samples": 8.0, | |
| "_scorer/specialty_id_score": 0.27, | |
| "_scorer/specialty_id_std": 0.1248, | |
| "_scorer/specialty_id_n_samples": 10.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 1.0, | |
| "_scorer/specialty_anesthesia_score": 0.5238, | |
| "_scorer/specialty_anesthesia_std": 0.1752, | |
| "_scorer/specialty_anesthesia_n_samples": 7.0, | |
| "_scorer/specialty_cards_score": 0.1818, | |
| "_scorer/specialty_cards_std": 0.1151, | |
| "_scorer/specialty_cards_n_samples": 11.0, | |
| "_scorer/specialty_derm_score": 0.0481, | |
| "_scorer/specialty_derm_std": 0.0469, | |
| "_scorer/specialty_derm_n_samples": 11.0, | |
| "_scorer/specialty_emergency_score": 0.2727, | |
| "_scorer/specialty_emergency_std": 0.1308, | |
| "_scorer/specialty_emergency_n_samples": 11.0, | |
| "_scorer/specialty_endo_score": 0.3429, | |
| "_scorer/specialty_endo_std": 0.1394, | |
| "_scorer/specialty_endo_n_samples": 10.0, | |
| "_scorer/specialty_genetics_score": 0.5, | |
| "_scorer/specialty_genetics_std": 0.2264, | |
| "_scorer/specialty_genetics_n_samples": 3.0, | |
| "_scorer/specialty_heme/onc_score": 0.1316, | |
| "_scorer/specialty_heme/onc_std": 0.0619, | |
| "_scorer/specialty_heme/onc_n_samples": 11.0, | |
| "_scorer/specialty_medicine_score": 0.1429, | |
| "_scorer/specialty_medicine_std": 0.1292, | |
| "_scorer/specialty_medicine_n_samples": 7.0, | |
| "_scorer/specialty_nephro_score": 0.5, | |
| "_scorer/specialty_nephro_std": 0.2551, | |
| "_scorer/specialty_nephro_n_samples": 4.0, | |
| "_scorer/specialty_neuro_score": 0.2622, | |
| "_scorer/specialty_neuro_std": 0.1331, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.4762, | |
| "_scorer/specialty_neurosurg_std": 0.17, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.4128, | |
| "_scorer/specialty_obgyn_std": 0.122, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 0.3333, | |
| "_scorer/specialty_ophtho_std": 0.2694, | |
| "_scorer/specialty_ophtho_n_samples": 3.0, | |
| "_scorer/specialty_ortho_score": 0.4286, | |
| "_scorer/specialty_ortho_std": 0.1876, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.1111, | |
| "_scorer/specialty_other_std": 0.1025, | |
| "_scorer/specialty_other_n_samples": 9.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 1.0, | |
| "_scorer/specialty_peds_score": 0.2111, | |
| "_scorer/specialty_peds_std": 0.1248, | |
| "_scorer/specialty_peds_n_samples": 10.0, | |
| "_scorer/specialty_primary_score": 0.4, | |
| "_scorer/specialty_primary_std": 0.2194, | |
| "_scorer/specialty_primary_n_samples": 5.0, | |
| "_scorer/specialty_psychiatry_score": 0.2222, | |
| "_scorer/specialty_psychiatry_std": 0.1348, | |
| "_scorer/specialty_psychiatry_n_samples": 9.0, | |
| "_scorer/specialty_pulm_score": 0.0, | |
| "_scorer/specialty_pulm_std": 0.0, | |
| "_scorer/specialty_pulm_n_samples": 5.0, | |
| "_scorer/specialty_rads_score": 0.4933, | |
| "_scorer/specialty_rads_std": 0.1963, | |
| "_scorer/specialty_rads_n_samples": 5.0, | |
| "_scorer/specialty_rheum_score": 0.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 4.0, | |
| "_scorer/specialty_surgery_score": 0.2857, | |
| "_scorer/specialty_surgery_std": 0.1707, | |
| "_scorer/specialty_surgery_n_samples": 7.0, | |
| "_scorer/specialty_uro_score": 0.0, | |
| "_scorer/specialty_uro_std": 0.0, | |
| "_scorer/specialty_uro_n_samples": 3.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2523, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0304, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": -0.0372, | |
| "healthbench_professional_score_length_adjusted/mean": -0.0687 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-5.4", | |
| "openrouter/deepseek/deepseek-v4-pro" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-5.4": 2312783, | |
| "openrouter/deepseek/deepseek-v4-pro": 1062156 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T20-28-47-00-00_healthbench-professional-physician-baseline_kjY8x2CXBcbHZUSHs55G7D.eval", | |
| "task": "healthbench_professional_physician_baseline", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T20:28:47+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4387, | |
| "_scorer/std": 0.0241, | |
| "_scorer/criteria_met_rate": 0.5044, | |
| "_scorer/use_case_consult_score": 0.4843, | |
| "_scorer/use_case_consult_std": 0.0285, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.59, | |
| "_scorer/use_case_research_std": 0.0304, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.449, | |
| "_scorer/use_case_writing_std": 0.0366, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.5561, | |
| "_scorer/type_good_faith_std": 0.0211, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.4137, | |
| "_scorer/type_red_teaming_std": 0.0314, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.5803, | |
| "_scorer/difficulty_typical_std": 0.0226, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.432, | |
| "_scorer/difficulty_difficult_std": 0.0279, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.4857, | |
| "_scorer/specialty_ent_std": 0.1788, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.5427, | |
| "_scorer/specialty_gi_std": 0.0926, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.5113, | |
| "_scorer/specialty_id_std": 0.0747, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.6229, | |
| "_scorer/specialty_anesthesia_std": 0.0871, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.4092, | |
| "_scorer/specialty_cards_std": 0.0638, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.6483, | |
| "_scorer/specialty_derm_std": 0.0777, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.4549, | |
| "_scorer/specialty_emergency_std": 0.1139, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5268, | |
| "_scorer/specialty_endo_std": 0.0838, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.7955, | |
| "_scorer/specialty_genetics_std": 0.1088, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.5019, | |
| "_scorer/specialty_heme/onc_std": 0.0829, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.6713, | |
| "_scorer/specialty_medicine_std": 0.097, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.4311, | |
| "_scorer/specialty_nephro_std": 0.0806, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.5592, | |
| "_scorer/specialty_neuro_std": 0.059, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.3242, | |
| "_scorer/specialty_neurosurg_std": 0.1121, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.4791, | |
| "_scorer/specialty_obgyn_std": 0.0634, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.3167, | |
| "_scorer/specialty_ophtho_std": 0.0981, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.3507, | |
| "_scorer/specialty_ortho_std": 0.0868, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.3541, | |
| "_scorer/specialty_other_std": 0.0869, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.6667, | |
| "_scorer/specialty_path_std": 0.2605, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.5363, | |
| "_scorer/specialty_peds_std": 0.0825, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.563, | |
| "_scorer/specialty_primary_std": 0.1593, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.5115, | |
| "_scorer/specialty_psychiatry_std": 0.0715, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.5848, | |
| "_scorer/specialty_pulm_std": 0.12, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.3787, | |
| "_scorer/specialty_rads_std": 0.1469, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.6964, | |
| "_scorer/specialty_rheum_std": 0.1457, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.498, | |
| "_scorer/specialty_surgery_std": 0.1162, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.6175, | |
| "_scorer/specialty_uro_std": 0.1148, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.5803, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0225, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.4768, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0516, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.4137, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0324, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.4429, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4387 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "physician_baseline_solver" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T23-41-11-00-00_healthbench-professional-consult_muhCA3DyeXvwNpDvn7gpN4.eval", | |
| "task": "healthbench_professional_consult", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 236, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 94, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T23:41:11+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1888, | |
| "total_samples": 1888, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.1778, | |
| "_scorer/std": 0.0344, | |
| "_scorer/criteria_met_rate": 0.465, | |
| "_scorer/use_case_consult_score": 0.3894, | |
| "_scorer/use_case_consult_std": 0.0288, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/type_good_faith_score": 0.4788, | |
| "_scorer/type_good_faith_std": 0.0361, | |
| "_scorer/type_good_faith_n_samples": 143.0, | |
| "_scorer/type_red_teaming_score": 0.2521, | |
| "_scorer/type_red_teaming_std": 0.042, | |
| "_scorer/type_red_teaming_n_samples": 93.0, | |
| "_scorer/difficulty_typical_score": 0.5822, | |
| "_scorer/difficulty_typical_std": 0.0444, | |
| "_scorer/difficulty_typical_n_samples": 84.0, | |
| "_scorer/difficulty_difficult_score": 0.2829, | |
| "_scorer/difficulty_difficult_std": 0.0349, | |
| "_scorer/difficulty_difficult_n_samples": 152.0, | |
| "_scorer/specialty_ent_score": 0.8271, | |
| "_scorer/specialty_ent_std": 0.0957, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.0, | |
| "_scorer/specialty_gi_std": 0.0, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.5251, | |
| "_scorer/specialty_id_std": 0.1189, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.3639, | |
| "_scorer/specialty_anesthesia_std": 0.1367, | |
| "_scorer/specialty_anesthesia_n_samples": 10.0, | |
| "_scorer/specialty_cards_score": 0.2994, | |
| "_scorer/specialty_cards_std": 0.0859, | |
| "_scorer/specialty_cards_n_samples": 23.0, | |
| "_scorer/specialty_derm_score": 0.4374, | |
| "_scorer/specialty_derm_std": 0.1206, | |
| "_scorer/specialty_derm_n_samples": 14.0, | |
| "_scorer/specialty_emergency_score": 0.6838, | |
| "_scorer/specialty_emergency_std": 0.1467, | |
| "_scorer/specialty_emergency_n_samples": 8.0, | |
| "_scorer/specialty_endo_score": 0.4092, | |
| "_scorer/specialty_endo_std": 0.1105, | |
| "_scorer/specialty_endo_n_samples": 9.0, | |
| "_scorer/specialty_genetics_score": 0.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.3, | |
| "_scorer/specialty_heme/onc_std": 0.1481, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.0778, | |
| "_scorer/specialty_medicine_std": 0.0723, | |
| "_scorer/specialty_medicine_n_samples": 6.0, | |
| "_scorer/specialty_nephro_score": 0.6452, | |
| "_scorer/specialty_nephro_std": 0.1375, | |
| "_scorer/specialty_nephro_n_samples": 9.0, | |
| "_scorer/specialty_neuro_score": 0.4072, | |
| "_scorer/specialty_neuro_std": 0.1084, | |
| "_scorer/specialty_neuro_n_samples": 15.0, | |
| "_scorer/specialty_neurosurg_score": 0.6801, | |
| "_scorer/specialty_neurosurg_std": 0.1509, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.3689, | |
| "_scorer/specialty_obgyn_std": 0.0948, | |
| "_scorer/specialty_obgyn_n_samples": 18.0, | |
| "_scorer/specialty_ophtho_score": 0.4089, | |
| "_scorer/specialty_ophtho_std": 0.1586, | |
| "_scorer/specialty_ophtho_n_samples": 9.0, | |
| "_scorer/specialty_ortho_score": 0.506, | |
| "_scorer/specialty_ortho_std": 0.1323, | |
| "_scorer/specialty_ortho_n_samples": 13.0, | |
| "_scorer/specialty_other_score": 0.3333, | |
| "_scorer/specialty_other_std": 0.1868, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.2222, | |
| "_scorer/specialty_peds_std": 0.1092, | |
| "_scorer/specialty_peds_n_samples": 14.0, | |
| "_scorer/specialty_primary_score": 0.6176, | |
| "_scorer/specialty_primary_std": 0.2088, | |
| "_scorer/specialty_primary_n_samples": 4.0, | |
| "_scorer/specialty_psychiatry_score": 0.4291, | |
| "_scorer/specialty_psychiatry_std": 0.1583, | |
| "_scorer/specialty_psychiatry_n_samples": 7.0, | |
| "_scorer/specialty_pulm_score": 0.3229, | |
| "_scorer/specialty_pulm_std": 0.1575, | |
| "_scorer/specialty_pulm_n_samples": 6.0, | |
| "_scorer/specialty_rads_score": 0.4286, | |
| "_scorer/specialty_rads_std": 0.1857, | |
| "_scorer/specialty_rads_n_samples": 7.0, | |
| "_scorer/specialty_rheum_score": 0.2524, | |
| "_scorer/specialty_rheum_std": 0.1416, | |
| "_scorer/specialty_rheum_n_samples": 5.0, | |
| "_scorer/specialty_surgery_score": 0.4, | |
| "_scorer/specialty_surgery_std": 0.1568, | |
| "_scorer/specialty_surgery_n_samples": 10.0, | |
| "_scorer/specialty_uro_score": 0.625, | |
| "_scorer/specialty_uro_std": 0.2167, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.5822, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0443, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 84.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3314, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0576, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2521, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0414, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, | |
| "healthbench_professional_score/mean": 0.2851, | |
| "healthbench_professional_score_length_adjusted/mean": 0.1778 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-27b-text-it", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-27b-text-it": 23572, | |
| "openai/gpt-5.4": 61467 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T23-41-59-00-00_healthbench-professional-writing_CMBT9aSmchzcQFRpwZdCqR.eval", | |
| "task": "healthbench_professional_writing", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 142, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 56, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T23:41:59+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1136, | |
| "total_samples": 1136, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0913, | |
| "_scorer/std": 0.0528, | |
| "_scorer/criteria_met_rate": 0.5106, | |
| "_scorer/use_case_writing_score": 0.3879, | |
| "_scorer/use_case_writing_std": 0.0371, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_red_teaming_score": 0.2182, | |
| "_scorer/type_red_teaming_std": 0.04, | |
| "_scorer/type_red_teaming_n_samples": 91.0, | |
| "_scorer/type_good_faith_score": 0.6907, | |
| "_scorer/type_good_faith_std": 0.0478, | |
| "_scorer/type_good_faith_n_samples": 51.0, | |
| "_scorer/difficulty_difficult_score": 0.2225, | |
| "_scorer/difficulty_difficult_std": 0.0394, | |
| "_scorer/difficulty_difficult_n_samples": 96.0, | |
| "_scorer/difficulty_typical_score": 0.7332, | |
| "_scorer/difficulty_typical_std": 0.0447, | |
| "_scorer/difficulty_typical_n_samples": 46.0, | |
| "_scorer/specialty_ent_score": 0.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 2.0, | |
| "_scorer/specialty_gi_score": 0.0, | |
| "_scorer/specialty_gi_std": 0.0, | |
| "_scorer/specialty_gi_n_samples": 5.0, | |
| "_scorer/specialty_id_score": 0.25, | |
| "_scorer/specialty_id_std": 0.1291, | |
| "_scorer/specialty_id_n_samples": 8.0, | |
| "_scorer/specialty_anesthesia_score": 0.4318, | |
| "_scorer/specialty_anesthesia_std": 0.2177, | |
| "_scorer/specialty_anesthesia_n_samples": 4.0, | |
| "_scorer/specialty_cards_score": 0.4634, | |
| "_scorer/specialty_cards_std": 0.1538, | |
| "_scorer/specialty_cards_n_samples": 6.0, | |
| "_scorer/specialty_derm_score": 0.2357, | |
| "_scorer/specialty_derm_std": 0.1489, | |
| "_scorer/specialty_derm_n_samples": 7.0, | |
| "_scorer/specialty_emergency_score": 0.3661, | |
| "_scorer/specialty_emergency_std": 0.1672, | |
| "_scorer/specialty_emergency_n_samples": 7.0, | |
| "_scorer/specialty_endo_score": 0.254, | |
| "_scorer/specialty_endo_std": 0.1405, | |
| "_scorer/specialty_endo_n_samples": 5.0, | |
| "_scorer/specialty_genetics_score": 1.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.477, | |
| "_scorer/specialty_heme/onc_std": 0.1453, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.3333, | |
| "_scorer/specialty_medicine_std": 0.2691, | |
| "_scorer/specialty_medicine_n_samples": 3.0, | |
| "_scorer/specialty_nephro_score": 0.6275, | |
| "_scorer/specialty_nephro_std": 0.1494, | |
| "_scorer/specialty_nephro_n_samples": 6.0, | |
| "_scorer/specialty_neuro_score": 0.3602, | |
| "_scorer/specialty_neuro_std": 0.1432, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.375, | |
| "_scorer/specialty_neurosurg_std": 0.2098, | |
| "_scorer/specialty_neurosurg_n_samples": 4.0, | |
| "_scorer/specialty_obgyn_score": 0.5261, | |
| "_scorer/specialty_obgyn_std": 0.1317, | |
| "_scorer/specialty_obgyn_n_samples": 12.0, | |
| "_scorer/specialty_ophtho_score": 0.5333, | |
| "_scorer/specialty_ophtho_std": 0.1997, | |
| "_scorer/specialty_ophtho_n_samples": 5.0, | |
| "_scorer/specialty_ortho_score": 0.5, | |
| "_scorer/specialty_ortho_std": 0.246, | |
| "_scorer/specialty_ortho_n_samples": 4.0, | |
| "_scorer/specialty_other_score": 0.2917, | |
| "_scorer/specialty_other_std": 0.1454, | |
| "_scorer/specialty_other_n_samples": 8.0, | |
| "_scorer/specialty_peds_score": 0.625, | |
| "_scorer/specialty_peds_std": 0.1471, | |
| "_scorer/specialty_peds_n_samples": 8.0, | |
| "_scorer/specialty_primary_score": 0.4902, | |
| "_scorer/specialty_primary_std": 0.2381, | |
| "_scorer/specialty_primary_n_samples": 3.0, | |
| "_scorer/specialty_psychiatry_score": 0.5028, | |
| "_scorer/specialty_psychiatry_std": 0.1346, | |
| "_scorer/specialty_psychiatry_n_samples": 12.0, | |
| "_scorer/specialty_pulm_score": 0.0, | |
| "_scorer/specialty_pulm_std": 0.0, | |
| "_scorer/specialty_pulm_n_samples": 2.0, | |
| "_scorer/specialty_rads_score": 0.0, | |
| "_scorer/specialty_rads_std": 0.0, | |
| "_scorer/specialty_rads_n_samples": 1.0, | |
| "_scorer/specialty_rheum_score": 0.5, | |
| "_scorer/specialty_rheum_std": 0.3484, | |
| "_scorer/specialty_rheum_n_samples": 2.0, | |
| "_scorer/specialty_surgery_score": 0.25, | |
| "_scorer/specialty_surgery_std": 0.2285, | |
| "_scorer/specialty_surgery_n_samples": 4.0, | |
| "_scorer/specialty_uro_score": 0.0, | |
| "_scorer/specialty_uro_std": 0.0, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2182, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0412, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.1763, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7332, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0439, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 46.0, | |
| "healthbench_professional_score/mean": 0.1891, | |
| "healthbench_professional_score_length_adjusted/mean": 0.0913 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-27b-text-it", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-27b-text-it": 5260, | |
| "openai/gpt-5.4": 19696 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T23-42-33-00-00_healthbench-professional-research_RFx9qF5ZP369Kdi3CNQU6M.eval", | |
| "task": "healthbench_professional_research", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 147, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 58, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T23:42:33+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1176, | |
| "total_samples": 1176, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.344, | |
| "_scorer/std": 0.0351, | |
| "_scorer/criteria_met_rate": 0.5214, | |
| "_scorer/use_case_research_score": 0.5213, | |
| "_scorer/use_case_research_std": 0.0339, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/type_red_teaming_score": 0.7143, | |
| "_scorer/type_red_teaming_std": 0.1773, | |
| "_scorer/type_red_teaming_n_samples": 7.0, | |
| "_scorer/type_good_faith_score": 0.5116, | |
| "_scorer/type_good_faith_std": 0.034, | |
| "_scorer/type_good_faith_n_samples": 140.0, | |
| "_scorer/difficulty_difficult_score": 0.451, | |
| "_scorer/difficulty_difficult_std": 0.1048, | |
| "_scorer/difficulty_difficult_n_samples": 21.0, | |
| "_scorer/difficulty_typical_score": 0.533, | |
| "_scorer/difficulty_typical_std": 0.0344, | |
| "_scorer/difficulty_typical_n_samples": 126.0, | |
| "_scorer/specialty_ent_score": 1.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 1.0, | |
| "_scorer/specialty_gi_score": 0.4415, | |
| "_scorer/specialty_gi_std": 0.1493, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.5615, | |
| "_scorer/specialty_id_std": 0.1441, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_anesthesia_score": 0.4464, | |
| "_scorer/specialty_anesthesia_std": 0.1392, | |
| "_scorer/specialty_anesthesia_n_samples": 8.0, | |
| "_scorer/specialty_cards_score": 0.5548, | |
| "_scorer/specialty_cards_std": 0.0978, | |
| "_scorer/specialty_cards_n_samples": 14.0, | |
| "_scorer/specialty_derm_score": 1.0, | |
| "_scorer/specialty_derm_std": 0.0, | |
| "_scorer/specialty_derm_n_samples": 4.0, | |
| "_scorer/specialty_emergency_score": 0.0, | |
| "_scorer/specialty_emergency_std": 0.0, | |
| "_scorer/specialty_emergency_n_samples": 1.0, | |
| "_scorer/specialty_endo_score": 0.94, | |
| "_scorer/specialty_endo_std": 0.0547, | |
| "_scorer/specialty_endo_n_samples": 6.0, | |
| "_scorer/specialty_genetics_score": 1.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 2.0, | |
| "_scorer/specialty_heme/onc_score": 0.2317, | |
| "_scorer/specialty_heme/onc_std": 0.0791, | |
| "_scorer/specialty_heme/onc_n_samples": 7.0, | |
| "_scorer/specialty_medicine_score": 0.625, | |
| "_scorer/specialty_medicine_std": 0.1735, | |
| "_scorer/specialty_medicine_n_samples": 8.0, | |
| "_scorer/specialty_nephro_score": 0.4507, | |
| "_scorer/specialty_nephro_std": 0.1604, | |
| "_scorer/specialty_nephro_n_samples": 7.0, | |
| "_scorer/specialty_neuro_score": 0.5824, | |
| "_scorer/specialty_neuro_std": 0.0874, | |
| "_scorer/specialty_neuro_n_samples": 19.0, | |
| "_scorer/specialty_neurosurg_score": 0.0, | |
| "_scorer/specialty_neurosurg_std": 0.0, | |
| "_scorer/specialty_neurosurg_n_samples": 1.0, | |
| "_scorer/specialty_obgyn_score": 0.3167, | |
| "_scorer/specialty_obgyn_std": 0.1306, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 0.625, | |
| "_scorer/specialty_ophtho_std": 0.2058, | |
| "_scorer/specialty_ophtho_n_samples": 4.0, | |
| "_scorer/specialty_ortho_score": 0.4238, | |
| "_scorer/specialty_ortho_std": 0.1465, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.5712, | |
| "_scorer/specialty_other_std": 0.1538, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_peds_score": 0.5003, | |
| "_scorer/specialty_peds_std": 0.1704, | |
| "_scorer/specialty_peds_n_samples": 6.0, | |
| "_scorer/specialty_psychiatry_score": 0.3217, | |
| "_scorer/specialty_psychiatry_std": 0.1213, | |
| "_scorer/specialty_psychiatry_n_samples": 10.0, | |
| "_scorer/specialty_pulm_score": 0.4224, | |
| "_scorer/specialty_pulm_std": 0.2187, | |
| "_scorer/specialty_pulm_n_samples": 4.0, | |
| "_scorer/specialty_rheum_score": 0.5714, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 1.0, | |
| "_scorer/specialty_surgery_score": 0.6556, | |
| "_scorer/specialty_surgery_std": 0.1419, | |
| "_scorer/specialty_surgery_n_samples": 3.0, | |
| "_scorer/specialty_uro_score": 0.5675, | |
| "_scorer/specialty_uro_std": 0.176, | |
| "_scorer/specialty_uro_n_samples": 5.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.7143, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.1676, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.533, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0367, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 126.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3193, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.1191, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, | |
| "healthbench_professional_score/mean": 0.4776, | |
| "healthbench_professional_score_length_adjusted/mean": 0.344 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-27b-text-it", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-27b-text-it": 12406, | |
| "openai/gpt-5.4": 45185 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T23-43-33-00-00_healthbench-professional-red-teaming_ewG7cuk2vXaZpcauRLz7qR.eval", | |
| "task": "healthbench_professional_red_teaming", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 191, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 76, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T23:43:33+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1528, | |
| "total_samples": 1528, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0, | |
| "_scorer/std": 0.0053, | |
| "_scorer/criteria_met_rate": 0.4126, | |
| "_scorer/use_case_consult_score": 0.2521, | |
| "_scorer/use_case_consult_std": 0.0418, | |
| "_scorer/use_case_consult_n_samples": 93.0, | |
| "_scorer/use_case_research_score": 0.7143, | |
| "_scorer/use_case_research_std": 0.1668, | |
| "_scorer/use_case_research_n_samples": 7.0, | |
| "_scorer/use_case_writing_score": 0.2182, | |
| "_scorer/use_case_writing_std": 0.0421, | |
| "_scorer/use_case_writing_n_samples": 91.0, | |
| "_scorer/type_red_teaming_score": 0.2529, | |
| "_scorer/type_red_teaming_std": 0.0292, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_difficult_score": 0.2529, | |
| "_scorer/difficulty_difficult_std": 0.0293, | |
| "_scorer/difficulty_difficult_n_samples": 191.0, | |
| "_scorer/specialty_ent_score": 0.5, | |
| "_scorer/specialty_ent_std": 0.2522, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.0, | |
| "_scorer/specialty_gi_std": 0.0, | |
| "_scorer/specialty_gi_n_samples": 8.0, | |
| "_scorer/specialty_id_score": 0.2471, | |
| "_scorer/specialty_id_std": 0.1041, | |
| "_scorer/specialty_id_n_samples": 10.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 1.0, | |
| "_scorer/specialty_anesthesia_score": 0.3274, | |
| "_scorer/specialty_anesthesia_std": 0.1669, | |
| "_scorer/specialty_anesthesia_n_samples": 7.0, | |
| "_scorer/specialty_cards_score": 0.3393, | |
| "_scorer/specialty_cards_std": 0.1297, | |
| "_scorer/specialty_cards_n_samples": 11.0, | |
| "_scorer/specialty_derm_score": 0.1818, | |
| "_scorer/specialty_derm_std": 0.1156, | |
| "_scorer/specialty_derm_n_samples": 11.0, | |
| "_scorer/specialty_emergency_score": 0.4148, | |
| "_scorer/specialty_emergency_std": 0.1421, | |
| "_scorer/specialty_emergency_n_samples": 11.0, | |
| "_scorer/specialty_endo_score": 0.2899, | |
| "_scorer/specialty_endo_std": 0.1183, | |
| "_scorer/specialty_endo_n_samples": 10.0, | |
| "_scorer/specialty_genetics_score": 0.6667, | |
| "_scorer/specialty_genetics_std": 0.2711, | |
| "_scorer/specialty_genetics_n_samples": 3.0, | |
| "_scorer/specialty_heme/onc_score": 0.3158, | |
| "_scorer/specialty_heme/onc_std": 0.1289, | |
| "_scorer/specialty_heme/onc_n_samples": 11.0, | |
| "_scorer/specialty_medicine_score": 0.1429, | |
| "_scorer/specialty_medicine_std": 0.1347, | |
| "_scorer/specialty_medicine_n_samples": 7.0, | |
| "_scorer/specialty_nephro_score": 0.75, | |
| "_scorer/specialty_nephro_std": 0.2103, | |
| "_scorer/specialty_nephro_n_samples": 4.0, | |
| "_scorer/specialty_neuro_score": 0.2622, | |
| "_scorer/specialty_neuro_std": 0.1385, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.3468, | |
| "_scorer/specialty_neurosurg_std": 0.1589, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.3852, | |
| "_scorer/specialty_obgyn_std": 0.1378, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 0.0, | |
| "_scorer/specialty_ophtho_std": 0.0, | |
| "_scorer/specialty_ophtho_n_samples": 3.0, | |
| "_scorer/specialty_ortho_score": 0.4286, | |
| "_scorer/specialty_ortho_std": 0.1936, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.0, | |
| "_scorer/specialty_other_std": 0.0, | |
| "_scorer/specialty_other_n_samples": 9.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 1.0, | |
| "_scorer/specialty_peds_score": 0.1111, | |
| "_scorer/specialty_peds_std": 0.0954, | |
| "_scorer/specialty_peds_n_samples": 10.0, | |
| "_scorer/specialty_primary_score": 0.4941, | |
| "_scorer/specialty_primary_std": 0.1993, | |
| "_scorer/specialty_primary_n_samples": 5.0, | |
| "_scorer/specialty_psychiatry_score": 0.1111, | |
| "_scorer/specialty_psychiatry_std": 0.1028, | |
| "_scorer/specialty_psychiatry_n_samples": 9.0, | |
| "_scorer/specialty_pulm_score": 0.2, | |
| "_scorer/specialty_pulm_std": 0.1756, | |
| "_scorer/specialty_pulm_n_samples": 5.0, | |
| "_scorer/specialty_rads_score": 0.2, | |
| "_scorer/specialty_rads_std": 0.1827, | |
| "_scorer/specialty_rads_n_samples": 5.0, | |
| "_scorer/specialty_rheum_score": 0.25, | |
| "_scorer/specialty_rheum_std": 0.2151, | |
| "_scorer/specialty_rheum_n_samples": 4.0, | |
| "_scorer/specialty_surgery_score": 0.0, | |
| "_scorer/specialty_surgery_std": 0.0, | |
| "_scorer/specialty_surgery_n_samples": 7.0, | |
| "_scorer/specialty_uro_score": 0.0, | |
| "_scorer/specialty_uro_std": 0.0, | |
| "_scorer/specialty_uro_n_samples": 3.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2529, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0302, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.0145, | |
| "healthbench_professional_score_length_adjusted/mean": -0.068 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-24T23-44-07-00-00_healthbench-professional-physician-baseline_cY3Z8sUNPRQTpSA3UHa77J.eval", | |
| "task": "healthbench_professional_physician_baseline", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 210, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-24T23:44:07+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249" | |
| }, | |
| "revision": null, | |
| "completed_samples": 4200, | |
| "total_samples": 4200, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4348, | |
| "_scorer/std": 0.023, | |
| "_scorer/criteria_met_rate": 0.5022, | |
| "_scorer/use_case_consult_score": 0.4758, | |
| "_scorer/use_case_consult_std": 0.0282, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.5942, | |
| "_scorer/use_case_research_std": 0.0317, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.4373, | |
| "_scorer/use_case_writing_std": 0.0364, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.5532, | |
| "_scorer/type_good_faith_std": 0.021, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.4029, | |
| "_scorer/type_red_teaming_std": 0.0336, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.5726, | |
| "_scorer/difficulty_typical_std": 0.0226, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.428, | |
| "_scorer/difficulty_difficult_std": 0.0269, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.4857, | |
| "_scorer/specialty_ent_std": 0.1752, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.5427, | |
| "_scorer/specialty_gi_std": 0.094, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.5113, | |
| "_scorer/specialty_id_std": 0.0726, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.6339, | |
| "_scorer/specialty_anesthesia_std": 0.0864, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.393, | |
| "_scorer/specialty_cards_std": 0.0608, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.6483, | |
| "_scorer/specialty_derm_std": 0.0758, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.4192, | |
| "_scorer/specialty_emergency_std": 0.1107, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5268, | |
| "_scorer/specialty_endo_std": 0.0865, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.6705, | |
| "_scorer/specialty_genetics_std": 0.2024, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.4844, | |
| "_scorer/specialty_heme/onc_std": 0.0841, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.6713, | |
| "_scorer/specialty_medicine_std": 0.1004, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.4879, | |
| "_scorer/specialty_nephro_std": 0.0756, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.5622, | |
| "_scorer/specialty_neuro_std": 0.0589, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.2723, | |
| "_scorer/specialty_neurosurg_std": 0.116, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.4284, | |
| "_scorer/specialty_obgyn_std": 0.0641, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.3167, | |
| "_scorer/specialty_ophtho_std": 0.0965, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.309, | |
| "_scorer/specialty_ortho_std": 0.0834, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.3395, | |
| "_scorer/specialty_other_std": 0.0885, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.6667, | |
| "_scorer/specialty_path_std": 0.2696, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.5363, | |
| "_scorer/specialty_peds_std": 0.084, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.563, | |
| "_scorer/specialty_primary_std": 0.1554, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.5277, | |
| "_scorer/specialty_psychiatry_std": 0.0702, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.5848, | |
| "_scorer/specialty_pulm_std": 0.1138, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.5037, | |
| "_scorer/specialty_rads_std": 0.1545, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.6964, | |
| "_scorer/specialty_rheum_std": 0.1513, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.498, | |
| "_scorer/specialty_surgery_std": 0.1186, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.6175, | |
| "_scorer/specialty_uro_std": 0.1097, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.5726, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0232, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.4897, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0533, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.4029, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0327, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.439, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4348 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-5.4": 17206029 | |
| }, | |
| "error": null, | |
| "solver": "physician_baseline_solver" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-25T00-59-37-00-00_healthbench-professional-consult_ffSg7GfyxjArrdwu7fSrBa.eval", | |
| "task": "healthbench_professional_consult", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "vllm/google/medgemma-4b-it", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 236, | |
| "epochs": 8, | |
| "limit": null, | |
| "config": { | |
| "epochs": 8, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 94, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-25T00:59:37+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.249" | |
| }, | |
| "revision": null, | |
| "completed_samples": 1888, | |
| "total_samples": 1888, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0819, | |
| "_scorer/std": 0.03, | |
| "_scorer/criteria_met_rate": 0.3351, | |
| "_scorer/use_case_consult_score": 0.2799, | |
| "_scorer/use_case_consult_std": 0.0253, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/type_good_faith_score": 0.3264, | |
| "_scorer/type_good_faith_std": 0.0331, | |
| "_scorer/type_good_faith_n_samples": 143.0, | |
| "_scorer/type_red_teaming_score": 0.2085, | |
| "_scorer/type_red_teaming_std": 0.0398, | |
| "_scorer/type_red_teaming_n_samples": 93.0, | |
| "_scorer/difficulty_typical_score": 0.4501, | |
| "_scorer/difficulty_typical_std": 0.0428, | |
| "_scorer/difficulty_typical_n_samples": 84.0, | |
| "_scorer/difficulty_difficult_score": 0.1859, | |
| "_scorer/difficulty_difficult_std": 0.0276, | |
| "_scorer/difficulty_difficult_n_samples": 152.0, | |
| "_scorer/specialty_ent_score": 0.7167, | |
| "_scorer/specialty_ent_std": 0.1381, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.0714, | |
| "_scorer/specialty_gi_std": 0.0687, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.1667, | |
| "_scorer/specialty_id_std": 0.1545, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.2292, | |
| "_scorer/specialty_anesthesia_std": 0.1276, | |
| "_scorer/specialty_anesthesia_n_samples": 10.0, | |
| "_scorer/specialty_cards_score": 0.304, | |
| "_scorer/specialty_cards_std": 0.0899, | |
| "_scorer/specialty_cards_n_samples": 23.0, | |
| "_scorer/specialty_derm_score": 0.3059, | |
| "_scorer/specialty_derm_std": 0.0956, | |
| "_scorer/specialty_derm_n_samples": 14.0, | |
| "_scorer/specialty_emergency_score": 0.6213, | |
| "_scorer/specialty_emergency_std": 0.1507, | |
| "_scorer/specialty_emergency_n_samples": 8.0, | |
| "_scorer/specialty_endo_score": 0.4684, | |
| "_scorer/specialty_endo_std": 0.0986, | |
| "_scorer/specialty_endo_n_samples": 9.0, | |
| "_scorer/specialty_genetics_score": 0.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.1, | |
| "_scorer/specialty_heme/onc_std": 0.0927, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.2537, | |
| "_scorer/specialty_medicine_std": 0.1017, | |
| "_scorer/specialty_medicine_n_samples": 6.0, | |
| "_scorer/specialty_nephro_score": 0.5461, | |
| "_scorer/specialty_nephro_std": 0.1395, | |
| "_scorer/specialty_nephro_n_samples": 9.0, | |
| "_scorer/specialty_neuro_score": 0.3117, | |
| "_scorer/specialty_neuro_std": 0.0877, | |
| "_scorer/specialty_neuro_n_samples": 15.0, | |
| "_scorer/specialty_neurosurg_score": 0.5388, | |
| "_scorer/specialty_neurosurg_std": 0.151, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.1789, | |
| "_scorer/specialty_obgyn_std": 0.0744, | |
| "_scorer/specialty_obgyn_n_samples": 18.0, | |
| "_scorer/specialty_ophtho_score": 0.2222, | |
| "_scorer/specialty_ophtho_std": 0.1368, | |
| "_scorer/specialty_ophtho_n_samples": 9.0, | |
| "_scorer/specialty_ortho_score": 0.3987, | |
| "_scorer/specialty_ortho_std": 0.1179, | |
| "_scorer/specialty_ortho_n_samples": 13.0, | |
| "_scorer/specialty_other_score": 0.1667, | |
| "_scorer/specialty_other_std": 0.1486, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.0714, | |
| "_scorer/specialty_peds_std": 0.0686, | |
| "_scorer/specialty_peds_n_samples": 14.0, | |
| "_scorer/specialty_primary_score": 0.5376, | |
| "_scorer/specialty_primary_std": 0.1833, | |
| "_scorer/specialty_primary_n_samples": 4.0, | |
| "_scorer/specialty_psychiatry_score": 0.2815, | |
| "_scorer/specialty_psychiatry_std": 0.1386, | |
| "_scorer/specialty_psychiatry_n_samples": 7.0, | |
| "_scorer/specialty_pulm_score": 0.0833, | |
| "_scorer/specialty_pulm_std": 0.0729, | |
| "_scorer/specialty_pulm_n_samples": 6.0, | |
| "_scorer/specialty_rads_score": 0.0714, | |
| "_scorer/specialty_rads_std": 0.0682, | |
| "_scorer/specialty_rads_n_samples": 7.0, | |
| "_scorer/specialty_rheum_score": 0.3524, | |
| "_scorer/specialty_rheum_std": 0.1962, | |
| "_scorer/specialty_rheum_n_samples": 5.0, | |
| "_scorer/specialty_surgery_score": 0.35, | |
| "_scorer/specialty_surgery_std": 0.1454, | |
| "_scorer/specialty_surgery_n_samples": 10.0, | |
| "_scorer/specialty_uro_score": 0.2026, | |
| "_scorer/specialty_uro_std": 0.1037, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.4501, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0412, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 84.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.1503, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0415, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2085, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0394, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, | |
| "healthbench_professional_score/mean": 0.152, | |
| "healthbench_professional_score_length_adjusted/mean": 0.0819 | |
| }, | |
| "model_usage_models": [ | |
| "vllm/google/medgemma-4b-it", | |
| "openai/gpt-5.4" | |
| ], | |
| "model_usage_total": { | |
| "vllm/google/medgemma-4b-it": 21234, | |
| "openai/gpt-5.4": 54391 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-25T18-26-20-00-00_healthbench-professional-consult_3DSH4qRFzYfb27rH9Z9RqP.eval", | |
| "task": "healthbench_professional_consult", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 236, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": 0.1, | |
| "continue_on_fail": false, | |
| "retry_on_error": 5, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-25T18:26:20+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 236, | |
| "total_samples": 236, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.2559, | |
| "_scorer/std": 0.0389, | |
| "_scorer/criteria_met_rate": 0.5483, | |
| "_scorer/use_case_consult_score": 0.4169, | |
| "_scorer/use_case_consult_std": 0.0278, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/type_good_faith_score": 0.531, | |
| "_scorer/type_good_faith_std": 0.0366, | |
| "_scorer/type_good_faith_n_samples": 143.0, | |
| "_scorer/type_red_teaming_score": 0.2414, | |
| "_scorer/type_red_teaming_std": 0.0413, | |
| "_scorer/type_red_teaming_n_samples": 93.0, | |
| "_scorer/difficulty_typical_score": 0.6563, | |
| "_scorer/difficulty_typical_std": 0.0403, | |
| "_scorer/difficulty_typical_n_samples": 84.0, | |
| "_scorer/difficulty_difficult_score": 0.2846, | |
| "_scorer/difficulty_difficult_std": 0.0343, | |
| "_scorer/difficulty_difficult_n_samples": 152.0, | |
| "_scorer/specialty_ent_score": 0.9437, | |
| "_scorer/specialty_ent_std": 0.0499, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.1429, | |
| "_scorer/specialty_gi_std": 0.1326, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.53, | |
| "_scorer/specialty_id_std": 0.1675, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.6928, | |
| "_scorer/specialty_anesthesia_std": 0.1297, | |
| "_scorer/specialty_anesthesia_n_samples": 10.0, | |
| "_scorer/specialty_cards_score": 0.267, | |
| "_scorer/specialty_cards_std": 0.0888, | |
| "_scorer/specialty_cards_n_samples": 23.0, | |
| "_scorer/specialty_derm_score": 0.5866, | |
| "_scorer/specialty_derm_std": 0.103, | |
| "_scorer/specialty_derm_n_samples": 14.0, | |
| "_scorer/specialty_emergency_score": 0.5224, | |
| "_scorer/specialty_emergency_std": 0.1573, | |
| "_scorer/specialty_emergency_n_samples": 8.0, | |
| "_scorer/specialty_endo_score": 0.5235, | |
| "_scorer/specialty_endo_std": 0.1395, | |
| "_scorer/specialty_endo_n_samples": 9.0, | |
| "_scorer/specialty_genetics_score": 0.5, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.05, | |
| "_scorer/specialty_heme/onc_std": 0.0483, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.0, | |
| "_scorer/specialty_medicine_std": 0.0, | |
| "_scorer/specialty_medicine_n_samples": 6.0, | |
| "_scorer/specialty_nephro_score": 0.3979, | |
| "_scorer/specialty_nephro_std": 0.1257, | |
| "_scorer/specialty_nephro_n_samples": 9.0, | |
| "_scorer/specialty_neuro_score": 0.5716, | |
| "_scorer/specialty_neuro_std": 0.0996, | |
| "_scorer/specialty_neuro_n_samples": 15.0, | |
| "_scorer/specialty_neurosurg_score": 0.7317, | |
| "_scorer/specialty_neurosurg_std": 0.1143, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.4717, | |
| "_scorer/specialty_obgyn_std": 0.0934, | |
| "_scorer/specialty_obgyn_n_samples": 18.0, | |
| "_scorer/specialty_ophtho_score": 0.2622, | |
| "_scorer/specialty_ophtho_std": 0.1367, | |
| "_scorer/specialty_ophtho_n_samples": 9.0, | |
| "_scorer/specialty_ortho_score": 0.5031, | |
| "_scorer/specialty_ortho_std": 0.1293, | |
| "_scorer/specialty_ortho_n_samples": 13.0, | |
| "_scorer/specialty_other_score": 0.3333, | |
| "_scorer/specialty_other_std": 0.1959, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.2222, | |
| "_scorer/specialty_peds_std": 0.1072, | |
| "_scorer/specialty_peds_n_samples": 14.0, | |
| "_scorer/specialty_primary_score": 0.5376, | |
| "_scorer/specialty_primary_std": 0.1921, | |
| "_scorer/specialty_primary_n_samples": 4.0, | |
| "_scorer/specialty_psychiatry_score": 0.572, | |
| "_scorer/specialty_psychiatry_std": 0.1603, | |
| "_scorer/specialty_psychiatry_n_samples": 7.0, | |
| "_scorer/specialty_pulm_score": 0.3229, | |
| "_scorer/specialty_pulm_std": 0.1496, | |
| "_scorer/specialty_pulm_n_samples": 6.0, | |
| "_scorer/specialty_rads_score": 0.3571, | |
| "_scorer/specialty_rads_std": 0.1658, | |
| "_scorer/specialty_rads_n_samples": 7.0, | |
| "_scorer/specialty_rheum_score": 0.5238, | |
| "_scorer/specialty_rheum_std": 0.2013, | |
| "_scorer/specialty_rheum_n_samples": 5.0, | |
| "_scorer/specialty_surgery_score": 0.5, | |
| "_scorer/specialty_surgery_std": 0.1601, | |
| "_scorer/specialty_surgery_n_samples": 10.0, | |
| "_scorer/specialty_uro_score": 0.5, | |
| "_scorer/specialty_uro_std": 0.2425, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.6563, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0386, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 84.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3526, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0587, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 59.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2414, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0408, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 93.0, | |
| "healthbench_professional_score/mean": 0.3123, | |
| "healthbench_professional_score_length_adjusted/mean": 0.2559 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-25T18-26-25-00-00_healthbench-professional-writing_2LJ8vJpJXUX2TbtokQzbxp.eval", | |
| "task": "healthbench_professional_writing", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 142, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": 0.1, | |
| "continue_on_fail": false, | |
| "retry_on_error": 5, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-25T18:26:25+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 142, | |
| "total_samples": 142, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0516, | |
| "_scorer/std": 0.0493, | |
| "_scorer/criteria_met_rate": 0.5729, | |
| "_scorer/use_case_writing_score": 0.3413, | |
| "_scorer/use_case_writing_std": 0.0356, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_red_teaming_score": 0.1317, | |
| "_scorer/type_red_teaming_std": 0.0337, | |
| "_scorer/type_red_teaming_n_samples": 91.0, | |
| "_scorer/type_good_faith_score": 0.7153, | |
| "_scorer/type_good_faith_std": 0.0482, | |
| "_scorer/type_good_faith_n_samples": 51.0, | |
| "_scorer/difficulty_difficult_score": 0.1457, | |
| "_scorer/difficulty_difficult_std": 0.033, | |
| "_scorer/difficulty_difficult_n_samples": 96.0, | |
| "_scorer/difficulty_typical_score": 0.7495, | |
| "_scorer/difficulty_typical_std": 0.0448, | |
| "_scorer/difficulty_typical_n_samples": 46.0, | |
| "_scorer/specialty_ent_score": 0.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 2.0, | |
| "_scorer/specialty_gi_score": 0.0, | |
| "_scorer/specialty_gi_std": 0.0, | |
| "_scorer/specialty_gi_n_samples": 5.0, | |
| "_scorer/specialty_id_score": 0.1875, | |
| "_scorer/specialty_id_std": 0.1245, | |
| "_scorer/specialty_id_n_samples": 8.0, | |
| "_scorer/specialty_anesthesia_score": 0.2545, | |
| "_scorer/specialty_anesthesia_std": 0.1292, | |
| "_scorer/specialty_anesthesia_n_samples": 4.0, | |
| "_scorer/specialty_cards_score": 0.625, | |
| "_scorer/specialty_cards_std": 0.1821, | |
| "_scorer/specialty_cards_n_samples": 6.0, | |
| "_scorer/specialty_derm_score": 0.1429, | |
| "_scorer/specialty_derm_std": 0.1288, | |
| "_scorer/specialty_derm_n_samples": 7.0, | |
| "_scorer/specialty_emergency_score": 0.1429, | |
| "_scorer/specialty_emergency_std": 0.1336, | |
| "_scorer/specialty_emergency_n_samples": 7.0, | |
| "_scorer/specialty_endo_score": 0.3474, | |
| "_scorer/specialty_endo_std": 0.1927, | |
| "_scorer/specialty_endo_n_samples": 5.0, | |
| "_scorer/specialty_genetics_score": 0.0, | |
| "_scorer/specialty_genetics_std": 0.0, | |
| "_scorer/specialty_genetics_n_samples": 1.0, | |
| "_scorer/specialty_heme/onc_score": 0.4362, | |
| "_scorer/specialty_heme/onc_std": 0.1471, | |
| "_scorer/specialty_heme/onc_n_samples": 10.0, | |
| "_scorer/specialty_medicine_score": 0.3333, | |
| "_scorer/specialty_medicine_std": 0.2676, | |
| "_scorer/specialty_medicine_n_samples": 3.0, | |
| "_scorer/specialty_nephro_score": 0.3914, | |
| "_scorer/specialty_nephro_std": 0.1664, | |
| "_scorer/specialty_nephro_n_samples": 6.0, | |
| "_scorer/specialty_neuro_score": 0.414, | |
| "_scorer/specialty_neuro_std": 0.1342, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.25, | |
| "_scorer/specialty_neurosurg_std": 0.2135, | |
| "_scorer/specialty_neurosurg_n_samples": 4.0, | |
| "_scorer/specialty_obgyn_score": 0.5399, | |
| "_scorer/specialty_obgyn_std": 0.1231, | |
| "_scorer/specialty_obgyn_n_samples": 12.0, | |
| "_scorer/specialty_ophtho_score": 0.5417, | |
| "_scorer/specialty_ophtho_std": 0.2039, | |
| "_scorer/specialty_ophtho_n_samples": 5.0, | |
| "_scorer/specialty_ortho_score": 0.25, | |
| "_scorer/specialty_ortho_std": 0.2196, | |
| "_scorer/specialty_ortho_n_samples": 4.0, | |
| "_scorer/specialty_other_score": 0.2917, | |
| "_scorer/specialty_other_std": 0.1496, | |
| "_scorer/specialty_other_n_samples": 8.0, | |
| "_scorer/specialty_peds_score": 0.6875, | |
| "_scorer/specialty_peds_std": 0.1536, | |
| "_scorer/specialty_peds_n_samples": 8.0, | |
| "_scorer/specialty_primary_score": 0.3333, | |
| "_scorer/specialty_primary_std": 0.2706, | |
| "_scorer/specialty_primary_n_samples": 3.0, | |
| "_scorer/specialty_psychiatry_score": 0.4611, | |
| "_scorer/specialty_psychiatry_std": 0.1256, | |
| "_scorer/specialty_psychiatry_n_samples": 12.0, | |
| "_scorer/specialty_pulm_score": 0.0, | |
| "_scorer/specialty_pulm_std": 0.0, | |
| "_scorer/specialty_pulm_n_samples": 2.0, | |
| "_scorer/specialty_rads_score": 0.4667, | |
| "_scorer/specialty_rads_std": 0.0, | |
| "_scorer/specialty_rads_n_samples": 1.0, | |
| "_scorer/specialty_rheum_score": 0.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 2.0, | |
| "_scorer/specialty_surgery_score": 0.25, | |
| "_scorer/specialty_surgery_std": 0.2181, | |
| "_scorer/specialty_surgery_n_samples": 4.0, | |
| "_scorer/specialty_uro_score": 0.0, | |
| "_scorer/specialty_uro_std": 0.0, | |
| "_scorer/specialty_uro_n_samples": 4.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.1317, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.034, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 91.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.4, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.2235, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 5.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.7495, | |
| "_scorer/source_slice_good_faith_typical_std": 0.045, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 46.0, | |
| "healthbench_professional_score/mean": 0.0973, | |
| "healthbench_professional_score_length_adjusted/mean": 0.0516 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-25T18-26-28-00-00_healthbench-professional-research_9Pb5iwuDu8dmuFzs5FnmBw.eval", | |
| "task": "healthbench_professional_research", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 147, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": 0.1, | |
| "continue_on_fail": false, | |
| "retry_on_error": 5, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-25T18:26:28+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 147, | |
| "total_samples": 147, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.5185, | |
| "_scorer/std": 0.0346, | |
| "_scorer/criteria_met_rate": 0.6757, | |
| "_scorer/use_case_research_score": 0.6383, | |
| "_scorer/use_case_research_std": 0.0316, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/type_red_teaming_score": 0.8571, | |
| "_scorer/type_red_teaming_std": 0.1322, | |
| "_scorer/type_red_teaming_n_samples": 7.0, | |
| "_scorer/type_good_faith_score": 0.6274, | |
| "_scorer/type_good_faith_std": 0.034, | |
| "_scorer/type_good_faith_n_samples": 140.0, | |
| "_scorer/difficulty_difficult_score": 0.5476, | |
| "_scorer/difficulty_difficult_std": 0.1009, | |
| "_scorer/difficulty_difficult_n_samples": 21.0, | |
| "_scorer/difficulty_typical_score": 0.6534, | |
| "_scorer/difficulty_typical_std": 0.0323, | |
| "_scorer/difficulty_typical_n_samples": 126.0, | |
| "_scorer/specialty_ent_score": 1.0, | |
| "_scorer/specialty_ent_std": 0.0, | |
| "_scorer/specialty_ent_n_samples": 1.0, | |
| "_scorer/specialty_gi_score": 0.4254, | |
| "_scorer/specialty_gi_std": 0.1646, | |
| "_scorer/specialty_gi_n_samples": 7.0, | |
| "_scorer/specialty_id_score": 0.5635, | |
| "_scorer/specialty_id_std": 0.1747, | |
| "_scorer/specialty_id_n_samples": 6.0, | |
| "_scorer/specialty_anesthesia_score": 0.6875, | |
| "_scorer/specialty_anesthesia_std": 0.1483, | |
| "_scorer/specialty_anesthesia_n_samples": 8.0, | |
| "_scorer/specialty_cards_score": 0.6477, | |
| "_scorer/specialty_cards_std": 0.0877, | |
| "_scorer/specialty_cards_n_samples": 14.0, | |
| "_scorer/specialty_derm_score": 0.75, | |
| "_scorer/specialty_derm_std": 0.2211, | |
| "_scorer/specialty_derm_n_samples": 4.0, | |
| "_scorer/specialty_emergency_score": 0.0, | |
| "_scorer/specialty_emergency_std": 0.0, | |
| "_scorer/specialty_emergency_n_samples": 1.0, | |
| "_scorer/specialty_endo_score": 0.7733, | |
| "_scorer/specialty_endo_std": 0.0922, | |
| "_scorer/specialty_endo_n_samples": 6.0, | |
| "_scorer/specialty_genetics_score": 0.8409, | |
| "_scorer/specialty_genetics_std": 0.1115, | |
| "_scorer/specialty_genetics_n_samples": 2.0, | |
| "_scorer/specialty_heme/onc_score": 0.651, | |
| "_scorer/specialty_heme/onc_std": 0.1332, | |
| "_scorer/specialty_heme/onc_n_samples": 7.0, | |
| "_scorer/specialty_medicine_score": 0.427, | |
| "_scorer/specialty_medicine_std": 0.1528, | |
| "_scorer/specialty_medicine_n_samples": 8.0, | |
| "_scorer/specialty_nephro_score": 0.8538, | |
| "_scorer/specialty_nephro_std": 0.0877, | |
| "_scorer/specialty_nephro_n_samples": 7.0, | |
| "_scorer/specialty_neuro_score": 0.7029, | |
| "_scorer/specialty_neuro_std": 0.0808, | |
| "_scorer/specialty_neuro_n_samples": 19.0, | |
| "_scorer/specialty_neurosurg_score": 0.0667, | |
| "_scorer/specialty_neurosurg_std": 0.0, | |
| "_scorer/specialty_neurosurg_n_samples": 1.0, | |
| "_scorer/specialty_obgyn_score": 0.5013, | |
| "_scorer/specialty_obgyn_std": 0.1234, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 1.0, | |
| "_scorer/specialty_ophtho_std": 0.0, | |
| "_scorer/specialty_ophtho_n_samples": 4.0, | |
| "_scorer/specialty_ortho_score": 0.3345, | |
| "_scorer/specialty_ortho_std": 0.1254, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.6267, | |
| "_scorer/specialty_other_std": 0.1434, | |
| "_scorer/specialty_other_n_samples": 6.0, | |
| "_scorer/specialty_peds_score": 0.7143, | |
| "_scorer/specialty_peds_std": 0.1761, | |
| "_scorer/specialty_peds_n_samples": 6.0, | |
| "_scorer/specialty_psychiatry_score": 0.6783, | |
| "_scorer/specialty_psychiatry_std": 0.1318, | |
| "_scorer/specialty_psychiatry_n_samples": 10.0, | |
| "_scorer/specialty_pulm_score": 0.5474, | |
| "_scorer/specialty_pulm_std": 0.1845, | |
| "_scorer/specialty_pulm_n_samples": 4.0, | |
| "_scorer/specialty_rheum_score": 1.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 1.0, | |
| "_scorer/specialty_surgery_score": 1.0, | |
| "_scorer/specialty_surgery_std": 0.0, | |
| "_scorer/specialty_surgery_n_samples": 3.0, | |
| "_scorer/specialty_uro_score": 0.5675, | |
| "_scorer/specialty_uro_std": 0.1738, | |
| "_scorer/specialty_uro_n_samples": 5.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.8571, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.1341, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 7.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.6534, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0325, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 126.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.3929, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.1225, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 14.0, | |
| "healthbench_professional_score/mean": 0.6298, | |
| "healthbench_professional_score_length_adjusted/mean": 0.5185 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-25T18-26-32-00-00_healthbench-professional-red-teaming_e9hCTjjRXrNTnexAxrYM8x.eval", | |
| "task": "healthbench_professional_red_teaming", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 191, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": 0.1, | |
| "continue_on_fail": false, | |
| "retry_on_error": 5, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-25T18:26:32+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 191, | |
| "total_samples": 191, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0, | |
| "_scorer/std": 0.0067, | |
| "_scorer/criteria_met_rate": 0.4643, | |
| "_scorer/use_case_consult_score": 0.2414, | |
| "_scorer/use_case_consult_std": 0.0408, | |
| "_scorer/use_case_consult_n_samples": 93.0, | |
| "_scorer/use_case_research_score": 0.8571, | |
| "_scorer/use_case_research_std": 0.1336, | |
| "_scorer/use_case_research_n_samples": 7.0, | |
| "_scorer/use_case_writing_score": 0.1317, | |
| "_scorer/use_case_writing_std": 0.0327, | |
| "_scorer/use_case_writing_n_samples": 91.0, | |
| "_scorer/type_red_teaming_score": 0.2117, | |
| "_scorer/type_red_teaming_std": 0.0287, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_difficult_score": 0.2117, | |
| "_scorer/difficulty_difficult_std": 0.0281, | |
| "_scorer/difficulty_difficult_n_samples": 191.0, | |
| "_scorer/specialty_ent_score": 0.5, | |
| "_scorer/specialty_ent_std": 0.2521, | |
| "_scorer/specialty_ent_n_samples": 4.0, | |
| "_scorer/specialty_gi_score": 0.0, | |
| "_scorer/specialty_gi_std": 0.0, | |
| "_scorer/specialty_gi_n_samples": 8.0, | |
| "_scorer/specialty_id_score": 0.25, | |
| "_scorer/specialty_id_std": 0.1261, | |
| "_scorer/specialty_id_n_samples": 10.0, | |
| "_scorer/specialty_allergy_score": 0.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 1.0, | |
| "_scorer/specialty_anesthesia_score": 0.5238, | |
| "_scorer/specialty_anesthesia_std": 0.1756, | |
| "_scorer/specialty_anesthesia_n_samples": 7.0, | |
| "_scorer/specialty_cards_score": 0.0909, | |
| "_scorer/specialty_cards_std": 0.0927, | |
| "_scorer/specialty_cards_n_samples": 11.0, | |
| "_scorer/specialty_derm_score": 0.1845, | |
| "_scorer/specialty_derm_std": 0.0962, | |
| "_scorer/specialty_derm_n_samples": 11.0, | |
| "_scorer/specialty_emergency_score": 0.1818, | |
| "_scorer/specialty_emergency_std": 0.1166, | |
| "_scorer/specialty_emergency_n_samples": 11.0, | |
| "_scorer/specialty_endo_score": 0.3429, | |
| "_scorer/specialty_endo_std": 0.1484, | |
| "_scorer/specialty_endo_n_samples": 10.0, | |
| "_scorer/specialty_genetics_score": 0.5, | |
| "_scorer/specialty_genetics_std": 0.2312, | |
| "_scorer/specialty_genetics_n_samples": 3.0, | |
| "_scorer/specialty_heme/onc_score": 0.2273, | |
| "_scorer/specialty_heme/onc_std": 0.1173, | |
| "_scorer/specialty_heme/onc_n_samples": 11.0, | |
| "_scorer/specialty_medicine_score": 0.1429, | |
| "_scorer/specialty_medicine_std": 0.1361, | |
| "_scorer/specialty_medicine_n_samples": 7.0, | |
| "_scorer/specialty_nephro_score": 0.0, | |
| "_scorer/specialty_nephro_std": 0.0, | |
| "_scorer/specialty_nephro_n_samples": 4.0, | |
| "_scorer/specialty_neuro_score": 0.1511, | |
| "_scorer/specialty_neuro_std": 0.109, | |
| "_scorer/specialty_neuro_n_samples": 9.0, | |
| "_scorer/specialty_neurosurg_score": 0.4762, | |
| "_scorer/specialty_neurosurg_std": 0.1635, | |
| "_scorer/specialty_neurosurg_n_samples": 6.0, | |
| "_scorer/specialty_obgyn_score": 0.4018, | |
| "_scorer/specialty_obgyn_std": 0.1363, | |
| "_scorer/specialty_obgyn_n_samples": 10.0, | |
| "_scorer/specialty_ophtho_score": 0.0, | |
| "_scorer/specialty_ophtho_std": 0.0, | |
| "_scorer/specialty_ophtho_n_samples": 3.0, | |
| "_scorer/specialty_ortho_score": 0.2857, | |
| "_scorer/specialty_ortho_std": 0.1683, | |
| "_scorer/specialty_ortho_n_samples": 7.0, | |
| "_scorer/specialty_other_score": 0.0, | |
| "_scorer/specialty_other_std": 0.0, | |
| "_scorer/specialty_other_n_samples": 9.0, | |
| "_scorer/specialty_path_score": 0.0, | |
| "_scorer/specialty_path_std": 0.0, | |
| "_scorer/specialty_path_n_samples": 1.0, | |
| "_scorer/specialty_peds_score": 0.2111, | |
| "_scorer/specialty_peds_std": 0.1252, | |
| "_scorer/specialty_peds_n_samples": 10.0, | |
| "_scorer/specialty_primary_score": 0.4, | |
| "_scorer/specialty_primary_std": 0.2224, | |
| "_scorer/specialty_primary_n_samples": 5.0, | |
| "_scorer/specialty_psychiatry_score": 0.2222, | |
| "_scorer/specialty_psychiatry_std": 0.1382, | |
| "_scorer/specialty_psychiatry_n_samples": 9.0, | |
| "_scorer/specialty_pulm_score": 0.0, | |
| "_scorer/specialty_pulm_std": 0.0, | |
| "_scorer/specialty_pulm_n_samples": 5.0, | |
| "_scorer/specialty_rads_score": 0.2933, | |
| "_scorer/specialty_rads_std": 0.1814, | |
| "_scorer/specialty_rads_n_samples": 5.0, | |
| "_scorer/specialty_rheum_score": 0.0, | |
| "_scorer/specialty_rheum_std": 0.0, | |
| "_scorer/specialty_rheum_n_samples": 4.0, | |
| "_scorer/specialty_surgery_score": 0.1429, | |
| "_scorer/specialty_surgery_std": 0.1288, | |
| "_scorer/specialty_surgery_n_samples": 7.0, | |
| "_scorer/specialty_uro_score": 0.0, | |
| "_scorer/specialty_uro_std": 0.0, | |
| "_scorer/specialty_uro_n_samples": 3.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.2117, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.028, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": -0.0532, | |
| "healthbench_professional_score_length_adjusted/mean": -0.0829 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "hs-prof-subsets", | |
| "file": "2026-07-25T18-26-35-00-00_healthbench-professional-physician-baseline_Njv8PLSvCJpAvUbPPMQqkx.eval", | |
| "task": "healthbench_professional_physician_baseline", | |
| "task_version": 0, | |
| "task_file": "healthbench_professional.py", | |
| "model": "openrouter/deepseek/deepseek-v4-pro", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "kwargs": { | |
| "judge_model": "openai/gpt-5.4", | |
| "judge_reasoning_effort": "low" | |
| } | |
| }, | |
| "dataset_samples": 525, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": 0.1, | |
| "continue_on_fail": false, | |
| "retry_on_error": 5, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-25T18:26:35+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.246" | |
| }, | |
| "revision": null, | |
| "completed_samples": 525, | |
| "total_samples": 525, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4289, | |
| "_scorer/std": 0.0236, | |
| "_scorer/criteria_met_rate": 0.5057, | |
| "_scorer/use_case_consult_score": 0.4776, | |
| "_scorer/use_case_consult_std": 0.0275, | |
| "_scorer/use_case_consult_n_samples": 236.0, | |
| "_scorer/use_case_research_score": 0.5781, | |
| "_scorer/use_case_research_std": 0.0331, | |
| "_scorer/use_case_research_n_samples": 147.0, | |
| "_scorer/use_case_writing_score": 0.4343, | |
| "_scorer/use_case_writing_std": 0.0345, | |
| "_scorer/use_case_writing_n_samples": 142.0, | |
| "_scorer/type_good_faith_score": 0.547, | |
| "_scorer/type_good_faith_std": 0.0211, | |
| "_scorer/type_good_faith_n_samples": 334.0, | |
| "_scorer/type_red_teaming_score": 0.4013, | |
| "_scorer/type_red_teaming_std": 0.0319, | |
| "_scorer/type_red_teaming_n_samples": 191.0, | |
| "_scorer/difficulty_typical_score": 0.5684, | |
| "_scorer/difficulty_typical_std": 0.0232, | |
| "_scorer/difficulty_typical_n_samples": 256.0, | |
| "_scorer/difficulty_difficult_score": 0.4232, | |
| "_scorer/difficulty_difficult_std": 0.0281, | |
| "_scorer/difficulty_difficult_n_samples": 269.0, | |
| "_scorer/specialty_ent_score": 0.4857, | |
| "_scorer/specialty_ent_std": 0.1722, | |
| "_scorer/specialty_ent_n_samples": 7.0, | |
| "_scorer/specialty_gi_score": 0.5953, | |
| "_scorer/specialty_gi_std": 0.0941, | |
| "_scorer/specialty_gi_n_samples": 19.0, | |
| "_scorer/specialty_id_score": 0.5394, | |
| "_scorer/specialty_id_std": 0.0696, | |
| "_scorer/specialty_id_n_samples": 20.0, | |
| "_scorer/specialty_allergy_score": 1.0, | |
| "_scorer/specialty_allergy_std": 0.0, | |
| "_scorer/specialty_allergy_n_samples": 2.0, | |
| "_scorer/specialty_anesthesia_score": 0.5969, | |
| "_scorer/specialty_anesthesia_std": 0.0908, | |
| "_scorer/specialty_anesthesia_n_samples": 22.0, | |
| "_scorer/specialty_cards_score": 0.393, | |
| "_scorer/specialty_cards_std": 0.0609, | |
| "_scorer/specialty_cards_n_samples": 43.0, | |
| "_scorer/specialty_derm_score": 0.6083, | |
| "_scorer/specialty_derm_std": 0.0803, | |
| "_scorer/specialty_derm_n_samples": 25.0, | |
| "_scorer/specialty_emergency_score": 0.4612, | |
| "_scorer/specialty_emergency_std": 0.1168, | |
| "_scorer/specialty_emergency_n_samples": 16.0, | |
| "_scorer/specialty_endo_score": 0.5108, | |
| "_scorer/specialty_endo_std": 0.0824, | |
| "_scorer/specialty_endo_n_samples": 20.0, | |
| "_scorer/specialty_genetics_score": 0.6705, | |
| "_scorer/specialty_genetics_std": 0.2035, | |
| "_scorer/specialty_genetics_n_samples": 4.0, | |
| "_scorer/specialty_heme/onc_score": 0.4659, | |
| "_scorer/specialty_heme/onc_std": 0.0861, | |
| "_scorer/specialty_heme/onc_n_samples": 27.0, | |
| "_scorer/specialty_medicine_score": 0.6713, | |
| "_scorer/specialty_medicine_std": 0.0984, | |
| "_scorer/specialty_medicine_n_samples": 17.0, | |
| "_scorer/specialty_nephro_score": 0.4311, | |
| "_scorer/specialty_nephro_std": 0.0782, | |
| "_scorer/specialty_nephro_n_samples": 22.0, | |
| "_scorer/specialty_neuro_score": 0.5338, | |
| "_scorer/specialty_neuro_std": 0.0636, | |
| "_scorer/specialty_neuro_n_samples": 43.0, | |
| "_scorer/specialty_neurosurg_score": 0.3242, | |
| "_scorer/specialty_neurosurg_std": 0.116, | |
| "_scorer/specialty_neurosurg_n_samples": 11.0, | |
| "_scorer/specialty_obgyn_score": 0.42, | |
| "_scorer/specialty_obgyn_std": 0.0637, | |
| "_scorer/specialty_obgyn_n_samples": 40.0, | |
| "_scorer/specialty_ophtho_score": 0.3167, | |
| "_scorer/specialty_ophtho_std": 0.0949, | |
| "_scorer/specialty_ophtho_n_samples": 18.0, | |
| "_scorer/specialty_ortho_score": 0.309, | |
| "_scorer/specialty_ortho_std": 0.081, | |
| "_scorer/specialty_ortho_n_samples": 24.0, | |
| "_scorer/specialty_other_score": 0.3541, | |
| "_scorer/specialty_other_std": 0.0886, | |
| "_scorer/specialty_other_n_samples": 20.0, | |
| "_scorer/specialty_path_score": 0.6667, | |
| "_scorer/specialty_path_std": 0.2659, | |
| "_scorer/specialty_path_n_samples": 3.0, | |
| "_scorer/specialty_peds_score": 0.5542, | |
| "_scorer/specialty_peds_std": 0.0832, | |
| "_scorer/specialty_peds_n_samples": 28.0, | |
| "_scorer/specialty_primary_score": 0.563, | |
| "_scorer/specialty_primary_std": 0.1594, | |
| "_scorer/specialty_primary_n_samples": 7.0, | |
| "_scorer/specialty_psychiatry_score": 0.5439, | |
| "_scorer/specialty_psychiatry_std": 0.072, | |
| "_scorer/specialty_psychiatry_n_samples": 29.0, | |
| "_scorer/specialty_pulm_score": 0.5848, | |
| "_scorer/specialty_pulm_std": 0.1168, | |
| "_scorer/specialty_pulm_n_samples": 12.0, | |
| "_scorer/specialty_rads_score": 0.3787, | |
| "_scorer/specialty_rads_std": 0.1462, | |
| "_scorer/specialty_rads_n_samples": 8.0, | |
| "_scorer/specialty_rheum_score": 0.6964, | |
| "_scorer/specialty_rheum_std": 0.1571, | |
| "_scorer/specialty_rheum_n_samples": 8.0, | |
| "_scorer/specialty_surgery_score": 0.498, | |
| "_scorer/specialty_surgery_std": 0.1167, | |
| "_scorer/specialty_surgery_n_samples": 17.0, | |
| "_scorer/specialty_uro_score": 0.6175, | |
| "_scorer/specialty_uro_std": 0.1148, | |
| "_scorer/specialty_uro_n_samples": 13.0, | |
| "_scorer/source_slice_good_faith_typical_score": 0.5684, | |
| "_scorer/source_slice_good_faith_typical_std": 0.0225, | |
| "_scorer/source_slice_good_faith_typical_n_samples": 256.0, | |
| "_scorer/source_slice_good_faith_difficult_score": 0.4768, | |
| "_scorer/source_slice_good_faith_difficult_std": 0.0535, | |
| "_scorer/source_slice_good_faith_difficult_n_samples": 78.0, | |
| "_scorer/source_slice_red_teaming_difficult_score": 0.4013, | |
| "_scorer/source_slice_red_teaming_difficult_std": 0.0314, | |
| "_scorer/source_slice_red_teaming_difficult_n_samples": 191.0, | |
| "healthbench_professional_score/mean": 0.4331, | |
| "healthbench_professional_score_length_adjusted/mean": 0.4289 | |
| }, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": null, | |
| "solver": "physician_baseline_solver" | |
| }, | |
| { | |
| "space": "medgemma-healthbench-hard", | |
| "file": "2026-08-05T06-05-04-00-00_healthbench-hard_U2qKVPncnWPTbmLEfZfa4y.eval", | |
| "task": "healthbench_hard", | |
| "task_version": 2, | |
| "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", | |
| "model": "openai-api/vllm/google/medgemma-27b-text-it", | |
| "model_args": {}, | |
| "model_generate_config": { | |
| "max_connections": 16, | |
| "max_tokens": 4096, | |
| "temperature": 0.0 | |
| }, | |
| "task_args": { | |
| "judge_model": "openai/gpt-4.1", | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "retry_on_error": 3, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-08-05T06:05:04+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.252" | |
| }, | |
| "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.1332, | |
| "_scorer/std": 0.0102, | |
| "_scorer/criteria_met_rate": 0.4424, | |
| "healthbench_score/mean": 0.1332, | |
| "healthbench_score_length_adjusted/mean": -0.0442 | |
| }, | |
| "model_usage_models": [ | |
| "openai-api/vllm/google/medgemma-27b-text-it", | |
| "openai/gpt-4.1" | |
| ], | |
| "model_usage_total": { | |
| "openai-api/vllm/google/medgemma-27b-text-it": 1108556, | |
| "openai/gpt-4.1": 22501745 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "medgemma-healthbench-hard", | |
| "file": "2026-08-05T07-09-21-00-00_healthbench-hard_4bZEuQss4Bbee8KixwTS4o.eval", | |
| "task": "healthbench_hard", | |
| "task_version": 2, | |
| "task_file": "inspect_evals/src/inspect_evals/healthbench/healthbench.py", | |
| "model": "openai-api/vllm/google/medgemma-4b-it", | |
| "model_args": {}, | |
| "model_generate_config": { | |
| "max_connections": 16, | |
| "max_tokens": 4096, | |
| "temperature": 0.0 | |
| }, | |
| "task_args": { | |
| "judge_model": "openai/gpt-4.1", | |
| "length_adjustment_center": 2000, | |
| "length_adjustment_penalty_per_500_chars": 0.0392 | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "retry_on_error": 3, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-08-05T07:09:21+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.252" | |
| }, | |
| "revision": "https://github.com/UKGovernmentBEIS/inspect_evals.git", | |
| "completed_samples": 1000, | |
| "total_samples": 1000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.0, | |
| "_scorer/std": 0.0, | |
| "_scorer/criteria_met_rate": 0.358, | |
| "healthbench_score/mean": -0.0312, | |
| "healthbench_score_length_adjusted/mean": -0.1641 | |
| }, | |
| "model_usage_models": [ | |
| "openai-api/vllm/google/medgemma-4b-it", | |
| "openai/gpt-4.1" | |
| ], | |
| "model_usage_total": { | |
| "openai-api/vllm/google/medgemma-4b-it": 918963, | |
| "openai/gpt-4.1": 20096686 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-v1", | |
| "file": "2026-07-09T08-07-45-00-00_healthbench-hard_VLbsW5zXdzSbfgeNMziFZ2.eval", | |
| "task": "inspect_evals/healthbench_hard", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai/gpt-5-nano", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": null, | |
| "length_adjustment_penalty_per_500_chars": null | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "error", | |
| "created": "2026-07-09T08:07:45+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.244", | |
| "inspect_evals": "0.14.2" | |
| }, | |
| "revision": "https://github.com/JonathanGabor/evilgenie_inspect.git", | |
| "completed_samples": null, | |
| "total_samples": null, | |
| "metrics": {}, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: \"sk-proj***************************************************************************************************************************", | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-v1", | |
| "file": "2026-07-09T08-10-01-00-00_healthbench-hard_jofngniV8jDhpdvbXSg9S4.eval", | |
| "task": "inspect_evals/healthbench_hard", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": null, | |
| "length_adjustment_penalty_per_500_chars": null | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "error", | |
| "created": "2026-07-09T08:10:01+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.244", | |
| "inspect_evals": "0.14.2" | |
| }, | |
| "revision": "https://github.com/JonathanGabor/evilgenie_inspect.git", | |
| "completed_samples": null, | |
| "total_samples": null, | |
| "metrics": {}, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: \"sk-proj***************************************************************************************************************************", | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-v1", | |
| "file": "2026-07-09T08-11-34-00-00_healthbench-hard_CzHRcCD6CoJpUFoxxWadFn.eval", | |
| "task": "inspect_evals/healthbench_hard", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "length_adjustment_center": null, | |
| "length_adjustment_penalty_per_500_chars": null | |
| }, | |
| "dataset_samples": 1000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 50, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "cancelled", | |
| "created": "2026-07-09T08:11:34+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.244", | |
| "inspect_evals": "0.14.2" | |
| }, | |
| "revision": "https://github.com/JonathanGabor/evilgenie_inspect.git", | |
| "completed_samples": null, | |
| "total_samples": null, | |
| "metrics": {}, | |
| "model_usage_models": [ | |
| "anthropic/claude-opus-4-7" | |
| ], | |
| "model_usage_total": { | |
| "anthropic/claude-opus-4-7": 46358 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-v1", | |
| "file": "2026-07-09T09-03-02-00-00_healthbench_EVLdZCZJQYG5EuWtS2TLsK.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4o-mini", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": null, | |
| "length_adjustment_penalty_per_500_chars": null | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "error", | |
| "created": "2026-07-09T09:03:02+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.245", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": null, | |
| "total_samples": null, | |
| "metrics": {}, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: \"sk-proj***************************************************************************************************************************", | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-v1", | |
| "file": "2026-07-09T09-05-32-00-00_healthbench_Aucfe2K5BgdpYrkukVNJcf.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4o-mini", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": null, | |
| "length_adjustment_penalty_per_500_chars": null | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "error", | |
| "created": "2026-07-09T09:05:32+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.245", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": null, | |
| "total_samples": null, | |
| "metrics": {}, | |
| "model_usage_models": [], | |
| "model_usage_total": {}, | |
| "error": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: \"sk-proj***************************************************************************************************************************", | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-v1", | |
| "file": "2026-07-09T09-09-45-00-00_healthbench_gGjNAEJ4UezLEQvJPVi7Zi.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "openai/gpt-5.5", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4o-mini", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": null, | |
| "length_adjustment_penalty_per_500_chars": null | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-09T09:09:45+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.245", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 5000, | |
| "total_samples": 5000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4873, | |
| "_scorer/std": 0.0045, | |
| "_scorer/criteria_met_rate": 0.4846, | |
| "healthbench_score/mean": 0.4873 | |
| }, | |
| "model_usage_models": [ | |
| "openai/gpt-5.5", | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "openai/gpt-5.5": 4699836, | |
| "openai/gpt-4o-mini": 84688797 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| }, | |
| { | |
| "space": "healthbench-v1", | |
| "file": "2026-07-09T10-33-03-00-00_healthbench_QeHrUjumcLfkEWL5T7HaUa.eval", | |
| "task": "inspect_evals/healthbench", | |
| "task_version": 2, | |
| "task_file": null, | |
| "model": "anthropic/claude-opus-4-7", | |
| "model_args": {}, | |
| "model_generate_config": {}, | |
| "task_args": { | |
| "subset": "full", | |
| "local_path": null, | |
| "judge_model": "openai/gpt-4o-mini", | |
| "judge_max_tokens": null, | |
| "judge_temperature": 0.0, | |
| "judge_system_message": "You are a helpful assistant.", | |
| "bootstrap_samples": 1000, | |
| "include_subset_scores": false, | |
| "length_adjustment_center": null, | |
| "length_adjustment_penalty_per_500_chars": null | |
| }, | |
| "dataset_samples": 5000, | |
| "epochs": 1, | |
| "limit": null, | |
| "config": { | |
| "epochs": 1, | |
| "fail_on_error": true, | |
| "continue_on_fail": false, | |
| "sandbox_cleanup": true, | |
| "log_samples": true, | |
| "log_realtime": true, | |
| "log_images": true, | |
| "log_buffer": 250, | |
| "log_shared": 0, | |
| "score_display": true | |
| }, | |
| "status": "success", | |
| "created": "2026-07-09T10:33:03+00:00", | |
| "inspect_version": { | |
| "inspect_ai": "0.3.245", | |
| "inspect_evals": "0.14.3" | |
| }, | |
| "revision": null, | |
| "completed_samples": 5000, | |
| "total_samples": 5000, | |
| "metrics": { | |
| "_scorer/bootstrap_score": 0.4762, | |
| "_scorer/std": 0.0043, | |
| "_scorer/criteria_met_rate": 0.4723, | |
| "healthbench_score/mean": 0.4762 | |
| }, | |
| "model_usage_models": [ | |
| "anthropic/claude-opus-4-7", | |
| "openai/gpt-4o-mini" | |
| ], | |
| "model_usage_total": { | |
| "anthropic/claude-opus-4-7": 4923085, | |
| "openai/gpt-4o-mini": 80274748 | |
| }, | |
| "error": null, | |
| "solver": "generate" | |
| } | |
| ] |