{
  "model_info": {
    "name": "Yi 34B",
    "id": "01-ai/yi-34b",
    "developer": "01-ai",
    "inference_platform": "unknown",
    "additional_details": {
      "model_id_aliases": [
        "01-ai/Yi-34B"
      ]
    }
  },
  "evaluations": [
    {
      "evaluation_id": "helm_lite/01-ai_yi-34b/1774096306.427425",
      "retrieved_timestamp": "1774096306.427425",
      "source_metadata": {
        "source_name": "helm_lite",
        "source_type": "documentation",
        "source_organization_name": "crfm",
        "evaluator_relationship": "third_party"
      },
      "eval_library": {
        "name": "helm",
        "version": "unknown"
      },
      "benchmark": "helm_lite",
      "evaluation_results": [
        {
          "evaluation_name": "Mean win rate",
          "source_data": {
            "dataset_name": "helm_lite",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "How many models this model outperforms on average (over columns).",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.57,
            "details": {
              "description": "",
              "tab": "Accuracy",
              "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.2681148564294632\"}",
              "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}"
            }
          },
          "generation_config": {
            "additional_details": {}
          }
        },
        {
          "evaluation_name": "NarrativeQA",
          "source_data": {
            "dataset_name": "NarrativeQA",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "F1 on NarrativeQA",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.782,
            "details": {
              "description": "min=0.782, mean=0.782, max=0.782, sum=0.782 (1)",
              "tab": "Accuracy",
              "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.368, mean=2.368, max=2.368, sum=2.368 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.368284817816506\"}",
              "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}",
              "NarrativeQA - # train": "{\"description\": \"min=4.868, mean=4.868, max=4.868, sum=4.868 (1)\", \"tab\": \"General information\", \"score\": \"4.867605633802817\"}",
              "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "NarrativeQA - # prompt tokens": "{\"description\": \"min=3611.445, mean=3611.445, max=3611.445, sum=3611.445 (1)\", \"tab\": \"General information\", \"score\": \"3611.445070422535\"}",
              "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {}
          }
        },
        {
          "evaluation_name": "NaturalQuestions (closed-book)",
          "source_data": {
            "dataset_name": "NaturalQuestions (closed-book)",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "F1 on NaturalQuestions (closed-book)",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.443,
            "details": {
              "description": "min=0.443, mean=0.443, max=0.443, sum=0.443 (1)",
              "tab": "Accuracy",
              "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.816, mean=1.816, max=1.816, sum=1.816 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.8157690076828004\"}",
              "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.458, mean=1.458, max=1.458, sum=1.458 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4578230485916137\"}",
              "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}",
              "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.838, mean=4.838, max=4.838, sum=4.838 (1)\", \"tab\": \"General information\", \"score\": \"4.838\"}",
              "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}",
              "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2171.698, mean=2171.698, max=2171.698, sum=2171.698 (1)\", \"tab\": \"General information\", \"score\": \"2171.698\"}",
              "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.995, mean=0.995, max=0.995, sum=0.995 (1)\", \"tab\": \"General information\", \"score\": \"0.995\"}",
              "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}",
              "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=131.695, mean=131.695, max=131.695, sum=131.695 (1)\", \"tab\": \"General information\", \"score\": \"131.695\"}",
              "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "mode": "\"closedbook\""
            }
          }
        },
        {
          "evaluation_name": "OpenbookQA",
          "source_data": {
            "dataset_name": "OpenbookQA",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on OpenbookQA",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.92,
            "details": {
              "description": "min=0.92, mean=0.92, max=0.92, sum=0.92 (1)",
              "tab": "Accuracy",
              "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.823, mean=0.823, max=0.823, sum=0.823 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8229070715904235\"}",
              "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}",
              "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "OpenbookQA - # prompt tokens": "{\"description\": \"min=260.002, mean=260.002, max=260.002, sum=260.002 (1)\", \"tab\": \"General information\", \"score\": \"260.002\"}",
              "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "dataset": "\"openbookqa\"",
              "method": "\"multiple_choice_joint\""
            }
          }
        },
        {
          "evaluation_name": "MMLU",
          "source_data": {
            "dataset_name": "MMLU",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on MMLU",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.65,
            "details": {
              "description": "min=0.4, mean=0.65, max=0.91, sum=3.248 (5)",
              "tab": "Accuracy",
              "MMLU - Observed inference time (s)": "{\"description\": \"min=0.511, mean=0.697, max=0.925, sum=3.486 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6972272023485417\"}",
              "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}",
              "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "MMLU - # prompt tokens": "{\"description\": \"min=383.67, mean=502.654, max=667.789, sum=2513.269 (5)\", \"tab\": \"General information\", \"score\": \"502.65389473684206\"}",
              "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]",
              "method": "\"multiple_choice_joint\""
            }
          }
        },
        {
          "evaluation_name": "MATH",
          "source_data": {
            "dataset_name": "MATH",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "Equivalent (CoT) on MATH",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.375,
            "details": {
              "description": "min=0.167, mean=0.375, max=0.563, sum=2.623 (7)",
              "tab": "Accuracy",
              "MATH - Observed inference time (s)": "{\"description\": \"min=2.651, mean=3.809, max=4.649, sum=26.664 (7)\", \"tab\": \"Efficiency\", \"score\": \"3.809198633421\"}",
              "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}",
              "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}",
              "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "MATH - # prompt tokens": "{\"description\": \"min=976.696, mean=1468.935, max=2582.038, sum=10282.547 (7)\", \"tab\": \"General information\", \"score\": \"1468.9352369693863\"}",
              "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]",
              "level": "\"1\"",
              "use_official_examples": "\"False\"",
              "use_chain_of_thought": "\"True\""
            }
          }
        },
        {
          "evaluation_name": "GSM8K",
          "source_data": {
            "dataset_name": "GSM8K",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on GSM8K",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.648,
            "details": {
              "description": "min=0.648, mean=0.648, max=0.648, sum=0.648 (1)",
              "tab": "Accuracy",
              "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.887, mean=4.887, max=4.887, sum=4.887 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.886563032150269\"}",
              "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}",
              "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "GSM8K - # prompt tokens": "{\"description\": \"min=1170.814, mean=1170.814, max=1170.814, sum=1170.814 (1)\", \"tab\": \"General information\", \"score\": \"1170.814\"}",
              "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {}
          }
        },
        {
          "evaluation_name": "LegalBench",
          "source_data": {
            "dataset_name": "LegalBench",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on LegalBench",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.618,
            "details": {
              "description": "min=0.311, mean=0.618, max=0.8, sum=3.089 (5)",
              "tab": "Accuracy",
              "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.465, mean=0.8, max=1.207, sum=4.002 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8004560962069804\"}",
              "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}",
              "LegalBench - # train": "{\"description\": \"min=2, mean=4.2, max=5, sum=21 (5)\", \"tab\": \"General information\", \"score\": \"4.2\"}",
              "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "LegalBench - # prompt tokens": "{\"description\": \"min=211.779, mean=951.524, max=3359.547, sum=4757.621 (5)\", \"tab\": \"General information\", \"score\": \"951.5242922438443\"}",
              "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]"
            }
          }
        },
        {
          "evaluation_name": "MedQA",
          "source_data": {
            "dataset_name": "MedQA",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on MedQA",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.656,
            "details": {
              "description": "min=0.656, mean=0.656, max=0.656, sum=0.656 (1)",
              "tab": "Accuracy",
              "MedQA - Observed inference time (s)": "{\"description\": \"min=1.064, mean=1.064, max=1.064, sum=1.064 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.064007310696672\"}",
              "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}",
              "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "MedQA - # prompt tokens": "{\"description\": \"min=1122.392, mean=1122.392, max=1122.392, sum=1122.392 (1)\", \"tab\": \"General information\", \"score\": \"1122.3916500994035\"}",
              "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {}
          }
        },
        {
          "evaluation_name": "WMT 2014",
          "source_data": {
            "dataset_name": "WMT 2014",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "BLEU-4 on WMT 2014",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.172,
            "details": {
              "description": "min=0.1, mean=0.172, max=0.218, sum=0.858 (5)",
              "tab": "Accuracy",
              "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.071, mean=1.404, max=2.506, sum=7.021 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.4042062711970469\"}",
              "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}",
              "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "WMT 2014 - # prompt tokens": "{\"description\": \"min=139.298, mean=187.092, max=317.56, sum=935.461 (5)\", \"tab\": \"General information\", \"score\": \"187.09213851506345\"}",
              "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]"
            }
          }
        }
      ],
      "detailed_evaluation_results": null,
      "generation_config": {
        "additional_details": {}
      }
    },
    {
      "evaluation_id": "helm_mmlu/01-ai_yi-34b/1774096312.00548",
      "retrieved_timestamp": "1774096312.00548",
      "source_metadata": {
        "source_name": "helm_mmlu",
        "source_type": "documentation",
        "source_organization_name": "crfm",
        "evaluator_relationship": "third_party"
      },
      "eval_library": {
        "name": "helm",
        "version": "unknown"
      },
      "benchmark": "helm_mmlu",
      "evaluation_results": [
        {
          "evaluation_name": "MMLU All Subjects",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on MMLU All Subjects",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.762,
            "details": {
              "description": "min=0.4, mean=0.762, max=0.974, sum=86.905 (114)",
              "tab": "Accuracy",
              "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.823, max=2.683, sum=93.841 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.8231679963633336\"}",
              "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}",
              "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=289.971, mean=661.842, max=2957.412, sum=75449.942 (114)\", \"tab\": \"General information\", \"score\": \"661.8416008681387\"}",
              "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]"
            }
          }
        },
        {
          "evaluation_name": "Abstract Algebra",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Abstract Algebra",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.4,
            "details": {
              "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)",
              "tab": "Accuracy",
              "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.658, mean=0.658, max=0.658, sum=1.315 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6577284264564515\"}",
              "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.67, mean=383.67, max=383.67, sum=767.34 (2)\", \"tab\": \"General information\", \"score\": \"383.67\"}",
              "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"abstract_algebra\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_abstract_algebra\""
            }
          }
        },
        {
          "evaluation_name": "Anatomy",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Anatomy",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.748,
            "details": {
              "description": "min=0.748, mean=0.748, max=0.748, sum=1.496 (2)",
              "tab": "Accuracy",
              "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.601, mean=0.601, max=0.601, sum=1.202 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6009190011907507\"}",
              "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}",
              "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Anatomy - # prompt tokens": "{\"description\": \"min=375.77, mean=375.77, max=375.77, sum=751.541 (2)\", \"tab\": \"General information\", \"score\": \"375.77037037037036\"}",
              "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"anatomy\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_anatomy\""
            }
          }
        },
        {
          "evaluation_name": "College Physics",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on College Physics",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.5,
            "details": {
              "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)",
              "tab": "Accuracy",
              "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.531, mean=0.531, max=0.531, sum=1.061 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5305842399597168\"}",
              "College Biology - Observed inference time (s)": "{\"description\": \"min=0.502, mean=0.502, max=0.502, sum=1.004 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5021488202942742\"}",
              "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.708, mean=0.708, max=0.708, sum=1.415 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7075318503379822\"}",
              "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.569, mean=0.569, max=0.569, sum=1.138 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5689087891578675\"}",
              "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.575, mean=0.575, max=0.575, sum=1.15 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5747669638925894\"}",
              "College Physics - Observed inference time (s)": "{\"description\": \"min=0.604, mean=0.604, max=0.604, sum=1.207 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.603668584543116\"}",
              "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "College Chemistry - # prompt tokens": "{\"description\": \"min=597.54, mean=597.54, max=597.54, sum=1195.08 (2)\", \"tab\": \"General information\", \"score\": \"597.54\"}",
              "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}",
              "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "College Biology - # prompt tokens": "{\"description\": \"min=514.819, mean=514.819, max=514.819, sum=1029.639 (2)\", \"tab\": \"General information\", \"score\": \"514.8194444444445\"}",
              "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "College Computer Science - # prompt tokens": "{\"description\": \"min=883.06, mean=883.06, max=883.06, sum=1766.12 (2)\", \"tab\": \"General information\", \"score\": \"883.06\"}",
              "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "College Mathematics - # prompt tokens": "{\"description\": \"min=635.3, mean=635.3, max=635.3, sum=1270.6 (2)\", \"tab\": \"General information\", \"score\": \"635.3\"}",
              "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}",
              "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "College Medicine - # prompt tokens": "{\"description\": \"min=549.688, mean=549.688, max=549.688, sum=1099.376 (2)\", \"tab\": \"General information\", \"score\": \"549.6878612716763\"}",
              "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}",
              "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "College Physics - # prompt tokens": "{\"description\": \"min=512.912, mean=512.912, max=512.912, sum=1025.824 (2)\", \"tab\": \"General information\", \"score\": \"512.9117647058823\"}",
              "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"college_physics\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_college_physics\""
            }
          }
        },
        {
          "evaluation_name": "Computer Security",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Computer Security",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.83,
            "details": {
              "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)",
              "tab": "Accuracy",
              "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.943 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47160084009170533\"}",
              "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Computer Security - # prompt tokens": "{\"description\": \"min=405.74, mean=405.74, max=405.74, sum=811.48 (2)\", \"tab\": \"General information\", \"score\": \"405.74\"}",
              "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"computer_security\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_computer_security\""
            }
          }
        },
        {
          "evaluation_name": "Econometrics",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Econometrics",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.588,
            "details": {
              "description": "min=0.588, mean=0.588, max=0.588, sum=1.175 (2)",
              "tab": "Accuracy",
              "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.61, mean=0.61, max=0.61, sum=1.219 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6095903463530958\"}",
              "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}",
              "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Econometrics - # prompt tokens": "{\"description\": \"min=667.789, mean=667.789, max=667.789, sum=1335.579 (2)\", \"tab\": \"General information\", \"score\": \"667.7894736842105\"}",
              "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"econometrics\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_econometrics\""
            }
          }
        },
        {
          "evaluation_name": "Global Facts",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Global Facts",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.53,
            "details": {
              "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)",
              "tab": "Accuracy",
              "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.537, mean=0.537, max=0.537, sum=1.074 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5369880175590516\"}",
              "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Global Facts - # prompt tokens": "{\"description\": \"min=462.32, mean=462.32, max=462.32, sum=924.64 (2)\", \"tab\": \"General information\", \"score\": \"462.32\"}",
              "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"global_facts\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_global_facts\""
            }
          }
        },
        {
          "evaluation_name": "Jurisprudence",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Jurisprudence",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.898,
            "details": {
              "description": "min=0.898, mean=0.898, max=0.898, sum=1.796 (2)",
              "tab": "Accuracy",
              "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.668, mean=0.668, max=0.668, sum=1.336 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.668224381075965\"}",
              "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}",
              "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Jurisprudence - # prompt tokens": "{\"description\": \"min=431.898, mean=431.898, max=431.898, sum=863.796 (2)\", \"tab\": \"General information\", \"score\": \"431.89814814814815\"}",
              "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"jurisprudence\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_jurisprudence\""
            }
          }
        },
        {
          "evaluation_name": "Philosophy",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Philosophy",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.82,
            "details": {
              "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)",
              "tab": "Accuracy",
              "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.848 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42395149779856395\"}",
              "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}",
              "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Philosophy - # prompt tokens": "{\"description\": \"min=356.723, mean=356.723, max=356.723, sum=713.447 (2)\", \"tab\": \"General information\", \"score\": \"356.7234726688103\"}",
              "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"philosophy\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_philosophy\""
            }
          }
        },
        {
          "evaluation_name": "Professional Psychology",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Professional Psychology",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.835,
            "details": {
              "description": "min=0.835, mean=0.835, max=0.835, sum=1.67 (2)",
              "tab": "Accuracy",
              "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=2.222, mean=2.222, max=2.222, sum=4.444 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.222188143169179\"}",
              "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=1.32 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6598629156748453\"}",
              "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.839, mean=1.839, max=1.839, sum=3.678 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.839003596032303\"}",
              "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=2.178, mean=2.178, max=2.178, sum=4.356 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1780028343200684\"}",
              "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}",
              "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Professional Medicine - # prompt tokens": "{\"description\": \"min=1202.533, mean=1202.533, max=1202.533, sum=2405.066 (2)\", \"tab\": \"General information\", \"score\": \"1202.5330882352941\"}",
              "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}",
              "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Professional Accounting - # prompt tokens": "{\"description\": \"min=771.16, mean=771.16, max=771.16, sum=1542.319 (2)\", \"tab\": \"General information\", \"score\": \"771.1595744680851\"}",
              "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}",
              "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Professional Law - # prompt tokens": "{\"description\": \"min=1759.098, mean=1759.098, max=1759.098, sum=3518.197 (2)\", \"tab\": \"General information\", \"score\": \"1759.0984354628422\"}",
              "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}",
              "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Professional Psychology - # prompt tokens": "{\"description\": \"min=608.201, mean=608.201, max=608.201, sum=1216.402 (2)\", \"tab\": \"General information\", \"score\": \"608.2009803921569\"}",
              "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"professional_psychology\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_professional_psychology\""
            }
          }
        },
        {
          "evaluation_name": "Us Foreign Policy",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Us Foreign Policy",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.91,
            "details": {
              "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)",
              "tab": "Accuracy",
              "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.607, mean=0.607, max=0.607, sum=1.214 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6068471717834473\"}",
              "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=458.53, mean=458.53, max=458.53, sum=917.06 (2)\", \"tab\": \"General information\", \"score\": \"458.53\"}",
              "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"us_foreign_policy\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_us_foreign_policy\""
            }
          }
        },
        {
          "evaluation_name": "Astronomy",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Astronomy",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.901,
            "details": {
              "description": "min=0.901, mean=0.901, max=0.901, sum=1.803 (2)",
              "tab": "Accuracy",
              "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.559, mean=0.559, max=0.559, sum=1.117 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5586237562330145\"}",
              "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}",
              "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Astronomy - # prompt tokens": "{\"description\": \"min=626.895, mean=626.895, max=626.895, sum=1253.789 (2)\", \"tab\": \"General information\", \"score\": \"626.8947368421053\"}",
              "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"astronomy\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_astronomy\""
            }
          }
        },
        {
          "evaluation_name": "Business Ethics",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Business Ethics",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.75,
            "details": {
              "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)",
              "tab": "Accuracy",
              "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=1.133 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5663742089271545\"}",
              "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Business Ethics - # prompt tokens": "{\"description\": \"min=616.97, mean=616.97, max=616.97, sum=1233.94 (2)\", \"tab\": \"General information\", \"score\": \"616.97\"}",
              "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"business_ethics\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_business_ethics\""
            }
          }
        },
        {
          "evaluation_name": "Clinical Knowledge",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Clinical Knowledge",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.8,
            "details": {
              "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)",
              "tab": "Accuracy",
              "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.487, mean=0.487, max=0.487, sum=0.975 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4874912774787759\"}",
              "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}",
              "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=446.966, mean=446.966, max=446.966, sum=893.932 (2)\", \"tab\": \"General information\", \"score\": \"446.96603773584906\"}",
              "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"clinical_knowledge\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_clinical_knowledge\""
            }
          }
        },
        {
          "evaluation_name": "Conceptual Physics",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Conceptual Physics",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.77,
            "details": {
              "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)",
              "tab": "Accuracy",
              "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4390637499220828\"}",
              "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}",
              "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Conceptual Physics - # prompt tokens": "{\"description\": \"min=311.94, mean=311.94, max=311.94, sum=623.881 (2)\", \"tab\": \"General information\", \"score\": \"311.9404255319149\"}",
              "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"conceptual_physics\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_conceptual_physics\""
            }
          }
        },
        {
          "evaluation_name": "Electrical Engineering",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Electrical Engineering",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.779,
            "details": {
              "description": "min=0.779, mean=0.779, max=0.779, sum=1.559 (2)",
              "tab": "Accuracy",
              "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.531, mean=0.531, max=0.531, sum=1.063 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.531287300175634\"}",
              "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}",
              "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Electrical Engineering - # prompt tokens": "{\"description\": \"min=491.993, mean=491.993, max=491.993, sum=983.986 (2)\", \"tab\": \"General information\", \"score\": \"491.99310344827586\"}",
              "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"electrical_engineering\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_electrical_engineering\""
            }
          }
        },
        {
          "evaluation_name": "Elementary Mathematics",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Elementary Mathematics",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.656,
            "details": {
              "description": "min=0.656, mean=0.656, max=0.656, sum=1.312 (2)",
              "tab": "Accuracy",
              "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.561, mean=0.561, max=0.561, sum=1.123 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5613514084033865\"}",
              "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}",
              "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=601.344, mean=601.344, max=601.344, sum=1202.688 (2)\", \"tab\": \"General information\", \"score\": \"601.3439153439153\"}",
              "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"elementary_mathematics\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_elementary_mathematics\""
            }
          }
        },
        {
          "evaluation_name": "Formal Logic",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Formal Logic",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.548,
            "details": {
              "description": "min=0.548, mean=0.548, max=0.548, sum=1.095 (2)",
              "tab": "Accuracy",
              "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.626, mean=0.626, max=0.626, sum=1.253 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6264226947511945\"}",
              "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}",
              "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Formal Logic - # prompt tokens": "{\"description\": \"min=675.579, mean=675.579, max=675.579, sum=1351.159 (2)\", \"tab\": \"General information\", \"score\": \"675.5793650793651\"}",
              "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"formal_logic\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_formal_logic\""
            }
          }
        },
        {
          "evaluation_name": "High School World History",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on High School World History",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.907,
            "details": {
              "description": "min=0.907, mean=0.907, max=0.907, sum=1.814 (2)",
              "tab": "Accuracy",
              "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.59, max=0.59, sum=1.179 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5895279146009876\"}",
              "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.562, mean=0.562, max=0.562, sum=1.124 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5618457112993512\"}",
              "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.851, mean=0.851, max=0.851, sum=1.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8510373497009277\"}",
              "High School European History - Observed inference time (s)": "{\"description\": \"min=2.359, mean=2.359, max=2.359, sum=4.717 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.358732930096713\"}",
              "High School Geography - Observed inference time (s)": "{\"description\": \"min=1.215, mean=1.215, max=1.215, sum=2.43 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.21489392266129\"}",
              "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.677, mean=0.677, max=0.677, sum=1.354 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6768323757487875\"}",
              "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.57, mean=0.57, max=0.57, sum=1.14 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5697616595488328\"}",
              "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.541, mean=0.541, max=0.541, sum=1.082 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5409333193743671\"}",
              "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.657, mean=0.657, max=0.657, sum=1.314 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6570467107436236\"}",
              "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.738, mean=0.738, max=0.738, sum=1.476 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7378138311651369\"}",
              "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.524, mean=0.524, max=0.524, sum=1.049 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5244918534515101\"}",
              "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.745, mean=0.745, max=0.745, sum=1.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7453252838717567\"}",
              "High School US History - Observed inference time (s)": "{\"description\": \"min=1.821, mean=1.821, max=1.821, sum=3.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.8211165923698276\"}",
              "High School World History - Observed inference time (s)": "{\"description\": \"min=1.27, mean=1.27, max=1.27, sum=2.541 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2703520537428714\"}",
              "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}",
              "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Biology - # prompt tokens": "{\"description\": \"min=546.394, mean=546.394, max=546.394, sum=1092.787 (2)\", \"tab\": \"General information\", \"score\": \"546.3935483870968\"}",
              "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}",
              "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Chemistry - # prompt tokens": "{\"description\": \"min=537.015, mean=537.015, max=537.015, sum=1074.03 (2)\", \"tab\": \"General information\", \"score\": \"537.0147783251232\"}",
              "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Computer Science - # prompt tokens": "{\"description\": \"min=962.1, mean=962.1, max=962.1, sum=1924.2 (2)\", \"tab\": \"General information\", \"score\": \"962.1\"}",
              "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}",
              "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School European History - # prompt tokens": "{\"description\": \"min=2957.412, mean=2957.412, max=2957.412, sum=5914.824 (2)\", \"tab\": \"General information\", \"score\": \"2957.4121212121213\"}",
              "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}",
              "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Geography - # prompt tokens": "{\"description\": \"min=404.035, mean=404.035, max=404.035, sum=808.071 (2)\", \"tab\": \"General information\", \"score\": \"404.0353535353535\"}",
              "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}",
              "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Government And Politics - # prompt tokens": "{\"description\": \"min=484.725, mean=484.725, max=484.725, sum=969.451 (2)\", \"tab\": \"General information\", \"score\": \"484.7253886010363\"}",
              "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}",
              "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=398.892, mean=398.892, max=398.892, sum=797.785 (2)\", \"tab\": \"General information\", \"score\": \"398.89230769230767\"}",
              "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}",
              "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Mathematics - # prompt tokens": "{\"description\": \"min=575.622, mean=575.622, max=575.622, sum=1151.244 (2)\", \"tab\": \"General information\", \"score\": \"575.6222222222223\"}",
              "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}",
              "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Microeconomics - # prompt tokens": "{\"description\": \"min=420.739, mean=420.739, max=420.739, sum=841.479 (2)\", \"tab\": \"General information\", \"score\": \"420.73949579831935\"}",
              "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}",
              "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Physics - # prompt tokens": "{\"description\": \"min=599.411, mean=599.411, max=599.411, sum=1198.821 (2)\", \"tab\": \"General information\", \"score\": \"599.4105960264901\"}",
              "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}",
              "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Psychology - # prompt tokens": "{\"description\": \"min=526.826, mean=526.826, max=526.826, sum=1053.651 (2)\", \"tab\": \"General information\", \"score\": \"526.8256880733945\"}",
              "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}",
              "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School Statistics - # prompt tokens": "{\"description\": \"min=869.778, mean=869.778, max=869.778, sum=1739.556 (2)\", \"tab\": \"General information\", \"score\": \"869.7777777777778\"}",
              "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}",
              "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School US History - # prompt tokens": "{\"description\": \"min=2369.132, mean=2369.132, max=2369.132, sum=4738.265 (2)\", \"tab\": \"General information\", \"score\": \"2369.1323529411766\"}",
              "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}",
              "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "High School World History - # prompt tokens": "{\"description\": \"min=1541.371, mean=1541.371, max=1541.371, sum=3082.743 (2)\", \"tab\": \"General information\", \"score\": \"1541.3713080168777\"}",
              "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"high_school_world_history\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_high_school_world_history\""
            }
          }
        },
        {
          "evaluation_name": "Human Sexuality",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Human Sexuality",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.87,
            "details": {
              "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)",
              "tab": "Accuracy",
              "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.768, mean=0.768, max=0.768, sum=1.535 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.76751750146327\"}",
              "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4077764613027791\"}",
              "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}",
              "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Human Aging - # prompt tokens": "{\"description\": \"min=332.013, mean=332.013, max=332.013, sum=664.027 (2)\", \"tab\": \"General information\", \"score\": \"332.0134529147982\"}",
              "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}",
              "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Human Sexuality - # prompt tokens": "{\"description\": \"min=367.855, mean=367.855, max=367.855, sum=735.71 (2)\", \"tab\": \"General information\", \"score\": \"367.85496183206106\"}",
              "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"human_sexuality\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_human_sexuality\""
            }
          }
        },
        {
          "evaluation_name": "International Law",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on International Law",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.909,
            "details": {
              "description": "min=0.909, mean=0.909, max=0.909, sum=1.818 (2)",
              "tab": "Accuracy",
              "International Law - Observed inference time (s)": "{\"description\": \"min=0.588, mean=0.588, max=0.588, sum=1.175 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5876634554429487\"}",
              "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}",
              "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "International Law - # prompt tokens": "{\"description\": \"min=663.289, mean=663.289, max=663.289, sum=1326.579 (2)\", \"tab\": \"General information\", \"score\": \"663.2892561983471\"}",
              "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"international_law\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_international_law\""
            }
          }
        },
        {
          "evaluation_name": "Logical Fallacies",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Logical Fallacies",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.883,
            "details": {
              "description": "min=0.883, mean=0.883, max=0.883, sum=1.767 (2)",
              "tab": "Accuracy",
              "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.622, mean=0.622, max=0.622, sum=1.245 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6223941814680041\"}",
              "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}",
              "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Logical Fallacies - # prompt tokens": "{\"description\": \"min=466.595, mean=466.595, max=466.595, sum=933.19 (2)\", \"tab\": \"General information\", \"score\": \"466.5950920245399\"}",
              "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"logical_fallacies\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_logical_fallacies\""
            }
          }
        },
        {
          "evaluation_name": "Machine Learning",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Machine Learning",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.58,
            "details": {
              "description": "min=0.58, mean=0.58, max=0.58, sum=1.161 (2)",
              "tab": "Accuracy",
              "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.638, mean=0.638, max=0.638, sum=1.277 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6384105682373047\"}",
              "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}",
              "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Machine Learning - # prompt tokens": "{\"description\": \"min=720.161, mean=720.161, max=720.161, sum=1440.321 (2)\", \"tab\": \"General information\", \"score\": \"720.1607142857143\"}",
              "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"machine_learning\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_machine_learning\""
            }
          }
        },
        {
          "evaluation_name": "Management",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Management",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.893,
            "details": {
              "description": "min=0.893, mean=0.893, max=0.893, sum=1.786 (2)",
              "tab": "Accuracy",
              "Management - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4204523748564489\"}",
              "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}",
              "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Management - # prompt tokens": "{\"description\": \"min=300.544, mean=300.544, max=300.544, sum=601.087 (2)\", \"tab\": \"General information\", \"score\": \"300.54368932038835\"}",
              "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"management\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_management\""
            }
          }
        },
        {
          "evaluation_name": "Marketing",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Marketing",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.936,
            "details": {
              "description": "min=0.936, mean=0.936, max=0.936, sum=1.872 (2)",
              "tab": "Accuracy",
              "Marketing - Observed inference time (s)": "{\"description\": \"min=0.463, mean=0.463, max=0.463, sum=0.926 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.463064443351876\"}",
              "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}",
              "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Marketing - # prompt tokens": "{\"description\": \"min=442.825, mean=442.825, max=442.825, sum=885.65 (2)\", \"tab\": \"General information\", \"score\": \"442.8247863247863\"}",
              "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"marketing\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_marketing\""
            }
          }
        },
        {
          "evaluation_name": "Medical Genetics",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Medical Genetics",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.87,
            "details": {
              "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)",
              "tab": "Accuracy",
              "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.857 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42836678981781007\"}",
              "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}",
              "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Medical Genetics - # prompt tokens": "{\"description\": \"min=362, mean=362, max=362, sum=724 (2)\", \"tab\": \"General information\", \"score\": \"362.0\"}",
              "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"medical_genetics\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_medical_genetics\""
            }
          }
        },
        {
          "evaluation_name": "Miscellaneous",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Miscellaneous",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.902,
            "details": {
              "description": "min=0.902, mean=0.902, max=0.902, sum=1.803 (2)",
              "tab": "Accuracy",
              "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.537, mean=0.537, max=0.537, sum=1.075 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5372742845333095\"}",
              "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}",
              "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Miscellaneous - # prompt tokens": "{\"description\": \"min=331.441, mean=331.441, max=331.441, sum=662.881 (2)\", \"tab\": \"General information\", \"score\": \"331.4406130268199\"}",
              "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"miscellaneous\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_miscellaneous\""
            }
          }
        },
        {
          "evaluation_name": "Moral Scenarios",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Moral Scenarios",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.606,
            "details": {
              "description": "min=0.606, mean=0.606, max=0.606, sum=1.211 (2)",
              "tab": "Accuracy",
              "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.671, mean=0.671, max=0.671, sum=1.341 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6705957754498961\"}",
              "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.764, mean=0.764, max=0.764, sum=1.528 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7642385613318928\"}",
              "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}",
              "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Moral Disputes - # prompt tokens": "{\"description\": \"min=507.913, mean=507.913, max=507.913, sum=1015.827 (2)\", \"tab\": \"General information\", \"score\": \"507.91329479768785\"}",
              "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}",
              "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}",
              "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Moral Scenarios - # prompt tokens": "{\"description\": \"min=703.334, mean=703.334, max=703.334, sum=1406.668 (2)\", \"tab\": \"General information\", \"score\": \"703.3340782122905\"}",
              "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"moral_scenarios\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_moral_scenarios\""
            }
          }
        },
        {
          "evaluation_name": "Nutrition",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Nutrition",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.869,
            "details": {
              "description": "min=0.869, mean=0.869, max=0.869, sum=1.739 (2)",
              "tab": "Accuracy",
              "Nutrition - Observed inference time (s)": "{\"description\": \"min=1.038, mean=1.038, max=1.038, sum=2.077 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0384757246067322\"}",
              "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}",
              "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Nutrition - # prompt tokens": "{\"description\": \"min=643.317, mean=643.317, max=643.317, sum=1286.634 (2)\", \"tab\": \"General information\", \"score\": \"643.3169934640523\"}",
              "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"nutrition\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_nutrition\""
            }
          }
        },
        {
          "evaluation_name": "Prehistory",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Prehistory",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.877,
            "details": {
              "description": "min=0.877, mean=0.877, max=0.877, sum=1.753 (2)",
              "tab": "Accuracy",
              "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.561, mean=0.561, max=0.561, sum=1.121 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.560588002204895\"}",
              "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}",
              "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Prehistory - # prompt tokens": "{\"description\": \"min=565.096, mean=565.096, max=565.096, sum=1130.191 (2)\", \"tab\": \"General information\", \"score\": \"565.0956790123457\"}",
              "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"prehistory\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_prehistory\""
            }
          }
        },
        {
          "evaluation_name": "Public Relations",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Public Relations",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.745,
            "details": {
              "description": "min=0.745, mean=0.745, max=0.745, sum=1.491 (2)",
              "tab": "Accuracy",
              "Public Relations - Observed inference time (s)": "{\"description\": \"min=2.107, mean=2.107, max=2.107, sum=4.213 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1067019375887783\"}",
              "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}",
              "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Public Relations - # prompt tokens": "{\"description\": \"min=432.436, mean=432.436, max=432.436, sum=864.873 (2)\", \"tab\": \"General information\", \"score\": \"432.43636363636364\"}",
              "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"public_relations\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_public_relations\""
            }
          }
        },
        {
          "evaluation_name": "Security Studies",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Security Studies",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.833,
            "details": {
              "description": "min=0.833, mean=0.833, max=0.833, sum=1.665 (2)",
              "tab": "Accuracy",
              "Security Studies - Observed inference time (s)": "{\"description\": \"min=2.683, mean=2.683, max=2.683, sum=5.366 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.682755525744691\"}",
              "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}",
              "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Security Studies - # prompt tokens": "{\"description\": \"min=1227.196, mean=1227.196, max=1227.196, sum=2454.392 (2)\", \"tab\": \"General information\", \"score\": \"1227.1959183673468\"}",
              "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"security_studies\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_security_studies\""
            }
          }
        },
        {
          "evaluation_name": "Sociology",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Sociology",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.9,
            "details": {
              "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)",
              "tab": "Accuracy",
              "Sociology - Observed inference time (s)": "{\"description\": \"min=1.401, mean=1.401, max=1.401, sum=2.803 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4013089469416224\"}",
              "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}",
              "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Sociology - # prompt tokens": "{\"description\": \"min=463.99, mean=463.99, max=463.99, sum=927.98 (2)\", \"tab\": \"General information\", \"score\": \"463.99004975124376\"}",
              "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"sociology\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_sociology\""
            }
          }
        },
        {
          "evaluation_name": "Virology",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on Virology",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.572,
            "details": {
              "description": "min=0.572, mean=0.572, max=0.572, sum=1.145 (2)",
              "tab": "Accuracy",
              "Virology - Observed inference time (s)": "{\"description\": \"min=0.563, mean=0.563, max=0.563, sum=1.127 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5633984617440098\"}",
              "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}",
              "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "Virology - # prompt tokens": "{\"description\": \"min=363.102, mean=363.102, max=363.102, sum=726.205 (2)\", \"tab\": \"General information\", \"score\": \"363.1024096385542\"}",
              "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"virology\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_virology\""
            }
          }
        },
        {
          "evaluation_name": "World Religions",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "EM on World Religions",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.877,
            "details": {
              "description": "min=0.877, mean=0.877, max=0.877, sum=1.754 (2)",
              "tab": "Accuracy",
              "World Religions - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.814 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4067504726655302\"}",
              "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}",
              "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}",
              "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}",
              "World Religions - # prompt tokens": "{\"description\": \"min=289.971, mean=289.971, max=289.971, sum=579.942 (2)\", \"tab\": \"General information\", \"score\": \"289.97076023391816\"}",
              "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}"
            }
          },
          "generation_config": {
            "additional_details": {
              "subject": "\"world_religions\"",
              "method": "\"multiple_choice_joint\"",
              "eval_split": "\"test\"",
              "groups": "\"mmlu_world_religions\""
            }
          }
        },
        {
          "evaluation_name": "Mean win rate",
          "source_data": {
            "dataset_name": "helm_mmlu",
            "source_type": "url",
            "url": [
              "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json"
            ]
          },
          "metric_config": {
            "evaluation_description": "How many models this model outperforms on average (over columns).",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.315,
            "details": {
              "description": "",
              "tab": "Efficiency"
            }
          },
          "generation_config": {
            "additional_details": {}
          }
        }
      ],
      "detailed_evaluation_results": null,
      "generation_config": {
        "additional_details": {
          "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]",
          "method": "\"multiple_choice_joint\"",
          "eval_split": "\"test\"",
          "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]"
        }
      }
    },
    {
      "evaluation_id": "hfopenllm_v2/01-ai_Yi-34B/1773936498.240187",
      "retrieved_timestamp": "1773936498.240187",
      "source_metadata": {
        "source_name": "HF Open LLM v2",
        "source_type": "documentation",
        "source_organization_name": "Hugging Face",
        "evaluator_relationship": "third_party"
      },
      "eval_library": {
        "name": "lm-evaluation-harness",
        "version": "0.4.0",
        "additional_details": {
          "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess"
        }
      },
      "benchmark": "hfopenllm_v2",
      "evaluation_results": [
        {
          "evaluation_name": "IFEval",
          "source_data": {
            "dataset_name": "IFEval",
            "source_type": "hf_dataset",
            "hf_repo": "google/IFEval"
          },
          "metric_config": {
            "evaluation_description": "Accuracy on IFEval",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.3046
          }
        },
        {
          "evaluation_name": "BBH",
          "source_data": {
            "dataset_name": "BBH",
            "source_type": "hf_dataset",
            "hf_repo": "SaylorTwift/bbh"
          },
          "metric_config": {
            "evaluation_description": "Accuracy on BBH",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.5457
          }
        },
        {
          "evaluation_name": "MATH Level 5",
          "source_data": {
            "dataset_name": "MATH Level 5",
            "source_type": "hf_dataset",
            "hf_repo": "DigitalLearningGmbH/MATH-lighteval"
          },
          "metric_config": {
            "evaluation_description": "Exact Match on MATH Level 5",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.0514
          }
        },
        {
          "evaluation_name": "GPQA",
          "source_data": {
            "dataset_name": "GPQA",
            "source_type": "hf_dataset",
            "hf_repo": "Idavidrein/gpqa"
          },
          "metric_config": {
            "evaluation_description": "Accuracy on GPQA",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.3666
          }
        },
        {
          "evaluation_name": "MUSR",
          "source_data": {
            "dataset_name": "MUSR",
            "source_type": "hf_dataset",
            "hf_repo": "TAUR-Lab/MuSR"
          },
          "metric_config": {
            "evaluation_description": "Accuracy on MUSR",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.4119
          }
        },
        {
          "evaluation_name": "MMLU-PRO",
          "source_data": {
            "dataset_name": "MMLU-PRO",
            "source_type": "hf_dataset",
            "hf_repo": "TIGER-Lab/MMLU-Pro"
          },
          "metric_config": {
            "evaluation_description": "Accuracy on MMLU-PRO",
            "lower_is_better": false,
            "score_type": "continuous",
            "min_score": 0.0,
            "max_score": 1.0
          },
          "score_details": {
            "score": 0.4412
          }
        }
      ],
      "detailed_evaluation_results": null,
      "generation_config": null
    }
  ]
}