feat(frontend): add differential evaluation support

- Dynamic Tasks generation from upstream llm-jp-eval yaml instead of
hardcoded enum, with local metric override for jculture-mcq (set_f1)
- Confirmation modal showing datasets grouped by category before submit
- Differential dataset computation: compare upstream dataset list against
existing parquet scores to determine missing datasets per model config
- Preview/confirm/cancel submission flow replacing single-step submit
- Partial evaluation display: rows with NaN scores are no longer hidden
- Extract upstream yaml fetching into src/upstream.py to avoid circular
imports
- Fix plot functions to handle missing AVG columns gracefully
- Add missing `requests` import in check_validity.py

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app.py

@@ -49,7 +49,7 @@ from src.i18n import (
     SELECT_NONE_BUTTON_LABEL_JA,
 )
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+from src.submission.submit import cancel_eval, confirm_eval, preview_eval
 
 
 def restart_space() -> None:
@@ -287,13 +287,14 @@ def plot_size_vs_score(df_filtered: pd.DataFrame) -> go.Figure:
         return go.Figure()
     df = ORIGINAL_DF[ORIGINAL_DF[AutoEvalColumn.row_id.name].isin(df_filtered[AutoEvalColumn.row_id.name])]
     df = df[df["#Params (B)"] > 0]
-    df = df[["model_name_for_query", "#Params (B)"] + AVG_COLUMNS]
+    available_avg = [c for c in AVG_COLUMNS if c in df.columns]
+    df = df[["model_name_for_query", "#Params (B)"] + available_avg]
     df = df.rename(columns={"model_name_for_query": "Model"})
     df["model_name_without_org_name"] = df["Model"].str.split("/").str[-1]
     df = pd.melt(
         df,
         id_vars=["Model", "model_name_without_org_name", "#Params (B)"],
-        value_vars=AVG_COLUMNS,
+        value_vars=available_avg,
         var_name="Category",
         value_name="Score",
     )
@@ -339,9 +340,10 @@ def plot_average_scores(df_filtered: pd.DataFrame) -> go.Figure:
     if len(ORIGINAL_DF) == 0 or AutoEvalColumn.row_id.name not in ORIGINAL_DF.columns:
         return go.Figure()
     df = ORIGINAL_DF[ORIGINAL_DF[AutoEvalColumn.row_id.name].isin(df_filtered[AutoEvalColumn.row_id.name])]
-    df = df[["model_name_for_query"] + list(TASK_AVG_NAME_MAP.keys())]
+    available_avg_keys = [k for k in TASK_AVG_NAME_MAP if k in df.columns]
+    df = df[["model_name_for_query"] + available_avg_keys]
     df = df.rename(columns={"model_name_for_query": "Model"})
-    df = df.rename(columns=TASK_AVG_NAME_MAP)
+    df = df.rename(columns={k: TASK_AVG_NAME_MAP[k] for k in available_avg_keys})
     df = df.set_index("Model")
 
     fig = go.Figure()
@@ -664,20 +666,42 @@ with gr.Blocks() as demo_submission:
 
     submit_button = gr.Button("Submit Eval")
     submission_result = gr.Markdown()
+    eval_entry_state = gr.State(value=None)
+
+    with gr.Column(visible=False, elem_classes="confirm-modal-overlay") as modal_overlay:
+        with gr.Column(elem_classes="confirm-modal-box"):
+            modal_content = gr.Markdown()
+            with gr.Row():
+                confirm_button = gr.Button("Confirm", variant="primary", scale=1)
+                cancel_button = gr.Button("Cancel", variant="secondary", scale=1)
+
+    submit_inputs = [
+        model_name_textbox,
+        revision_name_textbox,
+        precision,
+        model_type,
+        add_special_tokens,
+        apply_chat_template,
+        enable_thinking,
+        reasoning_parser,
+    ]
+
     submit_button.click(
-        fn=add_new_eval,
-        inputs=[
-            model_name_textbox,
-            revision_name_textbox,
-            precision,
-            model_type,
-            add_special_tokens,
-            apply_chat_template,
-            enable_thinking,
-            reasoning_parser,
-        ],
-        outputs=submission_result,
-        api_name="submit_eval",
+        fn=preview_eval,
+        inputs=submit_inputs,
+        outputs=[submission_result, modal_content, modal_overlay, eval_entry_state],
+        api_name="preview_eval",
+    )
+    confirm_button.click(
+        fn=confirm_eval,
+        inputs=[eval_entry_state],
+        outputs=[submission_result, modal_overlay, eval_entry_state],
+        api_name="confirm_eval",
+    )
+    cancel_button.click(
+        fn=cancel_eval,
+        inputs=[],
+        outputs=[submission_result, modal_overlay, eval_entry_state],
     )
 
     # Load user info when the page loads

src/about.py

@@ -1,6 +1,8 @@
 from dataclasses import dataclass
 from enum import Enum
 
+from src.upstream import load_upstream_eval_config
+
 
 class TaskType(Enum):
     AVG = "Average - 平均"
@@ -12,16 +14,103 @@ class TaskType(Enum):
     FA = "FA - 基礎分析"
     MR = "MR - 数学的推論"
     MT = "MT - 機械翻訳"
-    STS = "STS - 意味的類似度"
     HE_EN = "HE-EN - 英語試験問題"
     HE_JA = "HE-JA - 日本語試験問題"
     CG = "CG - コード生成"
     SUM = "SUM - 要約"
     BBH = "BBH - Big-Bench Hard"
     IF = "IF - 指示追従"
+    LM = "LM - 言語モデリング"
     NotTask = "?"
 
 
+# yaml category 名 → TaskType の対応
+_CATEGORY_TO_TASK_TYPE: dict[str, TaskType] = {
+    "NLI": TaskType.NLI,
+    "QA": TaskType.QA,
+    "RC": TaskType.RC,
+    "CR": TaskType.CR,
+    "EL": TaskType.EL,
+    "FA": TaskType.FA,
+    "MR": TaskType.MR,
+    "MT": TaskType.MT,
+    "HE-EN": TaskType.HE_EN,
+    "HE-JA": TaskType.HE_JA,
+    "CG": TaskType.CG,
+    "SUM": TaskType.SUM,
+    "BBH": TaskType.BBH,
+    "IF": TaskType.IF,
+    "LM": TaskType.LM,
+}
+
+# dataset 名 → 表示名のオーバーライド (ここに無い dataset は _auto_display_name で生成)
+_DISPLAY_NAME_OVERRIDES: dict[str, str] = {
+    "aime2024": "AIME 2024",
+    "aime2025": "AIME 2025",
+    "aio": "JAQKET",
+    "alt-e-to-j": "ALT E to J",
+    "alt-j-to-e": "ALT J to E",
+    "bigbenchhard_cot": "BBH CoT",
+    "bigbenchhard_direct": "BBH Direct",
+    "bigbenchhard_ja_cot": "BBH JA CoT",
+    "bigbenchhard_ja_direct": "BBH JA Direct",
+    "chabsa": "ChABSA",
+    "commonsensemoralja": "CommonSenseMoralJA",
+    "drop": "DROP",
+    "gpqa_diamond_en": "GPQA Diamond EN",
+    "gpqa_diamond_ja": "GPQA Diamond JA",
+    "gpqa_extended_en": "GPQA Extended EN",
+    "gpqa_extended_ja": "GPQA Extended JA",
+    "gpqa_main_en": "GPQA Main EN",
+    "gpqa_main_ja": "GPQA Main JA",
+    "gsm8k": "GSM8K",
+    "hle": "HLE",
+    "jamc-qa": "JAMC-QA",
+    "jamp": "JAMP",
+    "janli": "JANLI",
+    "jcommonsenseqa": "JCommonSenseQA",
+    "jemhopqa": "JEMHopQA",
+    "jhumaneval": "JHumanEval",
+    "jhle": "JHLE",
+    "jmmlu": "JMMLU",
+    "jnli": "JNLI",
+    "jsem": "JSEM",
+    "jsick": "JSICK",
+    "jsquad": "JSQuAD",
+    "kuci": "KUCI",
+    "mawps": "MAWPS",
+    "mbpp": "MBPP",
+    "mgsm": "MGSM",
+    "mif_eval_en": "MIF Eval EN",
+    "mif_eval_ja": "MIF Eval JA",
+    "mmlu_en": "MMLU",
+    "mmlu_prox_en": "MMLU Prox EN",
+    "mmlu_prox_ja": "MMLU Prox JA",
+    "mmmlu": "MMMLU",
+    "niilc": "NIILC",
+    "openbookqa": "OpenBookQA",
+    "polymath-en": "Polymath EN",
+    "polymath-ja": "Polymath JA",
+    "triviaqa": "TriviaQA",
+    "wiki_coreference": "Wiki Coreference",
+    "wiki_dependency": "Wiki Dependency",
+    "wiki_ner": "Wiki NER",
+    "wiki_pas": "Wiki PAS",
+    "wiki_reading": "Wiki Reading",
+    "wikicorpus-e-to-j": "WikiCorpus E to J",
+    "wikicorpus-j-to-e": "WikiCorpus J to E",
+    "winogrande_xl": "WinoGrande XL",
+    "xlsum_ja": "XL-Sum JA",
+    "jfinqa": "JFinQA",
+    "structeval": "StructEval",
+    "jculture-mcq": "JCulture-MCQ",
+    "jblimp": "JBLiMP",
+    "jcola-in-domain": "JCoLA (In-Domain)",
+    "jcola-out-of-domain": "JCoLA (Out-of-Domain)",
+    "jsts": "JSTS",
+}
+
+
 @dataclass
 class Task:
     benchmark: str
@@ -31,136 +120,43 @@ class Task:
     average: bool = False
 
 
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    AVG = Task("scores", "AVG", "AVG", TaskType.AVG, True)
-    NLI = Task("scores", "NLI", "AVG (NLI)", TaskType.NLI, True)  # Natural Language Inference - 自然言語推論
-    QA = Task("scores", "QA", "AVG (QA)", TaskType.QA, True)  # Question Answering - 質問応答
-    RC = Task("scores", "RC", "AVG (RC)", TaskType.RC, True)  # Reading Comprehension - 文章読解
-    EL = Task("scores", "EL", "AVG (EL)", TaskType.EL, True)  # Entity Linking - エンティティリンキング
-    FA = Task("scores", "FA", "AVG (FA)", TaskType.FA, True)  # Fundamental Analysis - 基礎解析
-    MR = Task("scores", "MR", "AVG (MR)", TaskType.MR, True)  # Mathematical Reasoning - 数学的推論
-    MT = Task("scores", "MT", "AVG (MT)", TaskType.MT, True)  # Machine Translation - 機械翻訳
-    HE_EN = Task("scores", "HE-EN", "AVG (HE-EN)", TaskType.HE_EN, True)  # Human Examination - English
-    HE_JA = Task("scores", "HE-JA", "AVG (HE-JA)", TaskType.HE_JA, True)  # Human Examination - Japanese
-    CG = Task("scores", "CG", "AVG (CG)", TaskType.CG, True)  # Code Generation - コード生成
-    SUM = Task("scores", "SUM", "AVG (SUM)", TaskType.SUM, True)  # Summarization - 要約
-    BBH = Task("scores", "BBH", "AVG (BBH)", TaskType.BBH, True)  # Big-Bench Hard
-    CR = Task("scores", "CR", "AVG (CR)", TaskType.CR, True)  # Commonsense Reasoning
-    IF = Task("scores", "IF", "AVG (IF)", TaskType.IF, True)  # Instruction Following
-    alt_e_to_j_bert_score_ja_f1 = Task("scores", "alt-e-to-j_bert_score_ja_f1", "ALT E to J BERT Score", TaskType.MT)
-    alt_e_to_j_bleu_ja = Task("scores", "alt-e-to-j_bleu_ja", "ALT E to J BLEU", TaskType.MT)
-    alt_e_to_j_comet_wmt22 = Task("scores", "alt-e-to-j_comet_wmt22", "ALT E to J COMET WMT22 ⭐", TaskType.MT)
-    alt_j_to_e_bert_score_en_f1 = Task("scores", "alt-j-to-e_bert_score_en_f1", "ALT J to E BERT Score", TaskType.MT)
-    alt_j_to_e_bleu_en = Task("scores", "alt-j-to-e_bleu_en", "ALT J to E BLEU", TaskType.MT)
-    alt_j_to_e_comet_wmt22 = Task("scores", "alt-j-to-e_comet_wmt22", "ALT J to E COMET WMT22 ⭐", TaskType.MT)
-    chabsa_set_f1 = Task("scores", "chabsa_set_f1", "ChABSA ⭐", TaskType.EL)
-    commonsensemoralja_exact_match = Task(
-        "scores", "commonsensemoralja_exact_match", "CommonSenseMoralJA ⭐", TaskType.CR
-    )
-    jamp_exact_match = Task("scores", "jamp_exact_match", "JAMP ⭐", TaskType.NLI)
-    janli_exact_match = Task("scores", "janli_exact_match", "JANLI ⭐", TaskType.NLI)
-    jcommonsenseqa_exact_match = Task("scores", "jcommonsenseqa_exact_match", "JCommonSenseQA ⭐", TaskType.CR)
-    jemhopqa_char_f1 = Task("scores", "jemhopqa_char_f1", "JEMHopQA ⭐", TaskType.QA)
-    jmmlu_exact_match = Task("scores", "jmmlu_exact_match", "JMMLU ⭐", TaskType.HE_JA)
-    jnli_exact_match = Task("scores", "jnli_exact_match", "JNLI ⭐", TaskType.NLI)
-    jsem_exact_match = Task("scores", "jsem_exact_match", "JSEM ⭐", TaskType.NLI)
-    jsick_exact_match = Task("scores", "jsick_exact_match", "JSICK ⭐", TaskType.NLI)
-    jsquad_char_f1 = Task("scores", "jsquad_char_f1", "JSquad ⭐", TaskType.RC)
-    jsts_pearson = Task(
-        "scores", "jsts_pearson", "JSTS (Pearson)", TaskType.STS
-    )  # Semantic Textual Similarity - 意味的類似度
-    jsts_spearman = Task(
-        "scores", "jsts_spearman", "JSTS (Spearman)", TaskType.STS
-    )  # Semantic Textual Similarity - 意味的類似度
-    kuci_exact_match = Task("scores", "kuci_exact_match", "KUCI ⭐", TaskType.CR)
-    mawps_mathematical_equivalence = Task("scores", "mawps_mathematical_equivalence", "MAWPS ⭐", TaskType.MR)
-    mbpp_code_exec_sandbox = Task("scores", "mbpp_code_exec_sandbox", "MBPP (exec) (0 shots only) ⭐", TaskType.CG)
-    mbpp_pylint_check = Task("scores", "mbpp_pylint_check", "MBPP (pylint) (0 shots only)", TaskType.CG)
-    mmlu_en_exact_match = Task("scores", "mmlu_en_exact_match", "MMLU ⭐", TaskType.HE_EN)
-    niilc_char_f1 = Task("scores", "niilc_char_f1", "NIILC ⭐", TaskType.QA)
-    aio_char_f1 = Task("scores", "aio_char_f1", "JAQKET ⭐", TaskType.QA)
-    wiki_coreference_set_f1 = Task("scores", "wiki_coreference_set_f1", "Wiki Coreference ⭐", TaskType.FA)
-    wiki_dependency_set_f1 = Task("scores", "wiki_dependency_set_f1", "Wiki Dependency ⭐", TaskType.FA)
-    wiki_ner_set_f1 = Task("scores", "wiki_ner_set_f1", "Wiki NER ⭐", TaskType.FA)
-    wiki_pas_set_f1 = Task("scores", "wiki_pas_set_f1", "Wiki PAS ⭐", TaskType.FA)
-    wiki_reading_char_f1 = Task("scores", "wiki_reading_char_f1", "Wiki Reading ⭐", TaskType.FA)
-    wikicorpus_e_to_j_bert_score_ja_f1 = Task(
-        "scores", "wikicorpus-e-to-j_bert_score_ja_f1", "WikiCorpus E to J BERT Score", TaskType.MT
-    )
-    wikicorpus_e_to_j_bleu_ja = Task("scores", "wikicorpus-e-to-j_bleu_ja", "WikiCorpus E to J BLEU", TaskType.MT)
-    wikicorpus_e_to_j_comet_wmt22 = Task(
-        "scores", "wikicorpus-e-to-j_comet_wmt22", "WikiCorpus E to J COMET WMT22 ⭐", TaskType.MT
-    )
-    wikicorpus_j_to_e_bert_score_en_f1 = Task(
-        "scores", "wikicorpus-j-to-e_bert_score_en_f1", "WikiCorpus J to E BERT Score", TaskType.MT
-    )
-    wikicorpus_j_to_e_bleu_en = Task("scores", "wikicorpus-j-to-e_bleu_en", "WikiCorpus J to E BLEU", TaskType.MT)
-    wikicorpus_j_to_e_comet_wmt22 = Task(
-        "scores", "wikicorpus-j-to-e_comet_wmt22", "WikiCorpus J to E COMET WMT22 ⭐", TaskType.MT
-    )
-    xlsum_ja_bert_score_ja_f1 = Task(
-        "scores", "xlsum_ja_bert_score_ja_f1", "XL-Sum JA BERT Score (0 shots only)", TaskType.SUM
-    )
-    xlsum_ja_bleu_ja = Task("scores", "xlsum_ja_bleu_ja", "XL-Sum JA BLEU (0 shots only)", TaskType.SUM)
-    xlsum_ja_rouge1 = Task("scores", "xlsum_ja_rouge1", "XL-Sum ROUGE1 (0 shots only)", TaskType.SUM)
-    xlsum_ja_rouge2 = Task("scores", "xlsum_ja_rouge2", "XL-Sum ROUGE2 (0 shots only) ⭐", TaskType.SUM)
-    # xlsum_ja_rouge2_scaling = Task("scores", "xlsum_ja_rouge2_scaling", "XL-Sum JA ROUGE2 Scaling")
-    xlsum_ja_rouge_lsum = Task("scores", "xlsum_ja_rougeLsum", "XL-Sum ROUGE-Lsum (0 shots only)", TaskType.SUM)
-    # New tasks for v2.0.0
-    aime2024_mathematical_equivalence = Task(
-        "scores", "aime2024_mathematical_equivalence", "AIME 2024 ⭐", TaskType.MR
-    )
-    aime2025_mathematical_equivalence = Task(
-        "scores", "aime2025_mathematical_equivalence", "AIME 2025 ⭐", TaskType.MR
-    )
-    bigbenchhard_direct_exact_match = Task("scores", "bigbenchhard_direct_exact_match", "BBH Direct ⭐", TaskType.BBH)
-    bigbenchhard_cot_exact_match = Task("scores", "bigbenchhard_cot_exact_match", "BBH CoT ⭐", TaskType.BBH)
-    bigbenchhard_ja_direct_exact_match = Task(
-        "scores", "bigbenchhard_ja_direct_exact_match", "BBH JA Direct ⭐", TaskType.BBH
-    )
-    bigbenchhard_ja_cot_exact_match = Task("scores", "bigbenchhard_ja_cot_exact_match", "BBH JA CoT ⭐", TaskType.BBH)
-    drop_drop_f1 = Task("scores", "drop_drop_f1", "DROP ⭐", TaskType.QA)
-    gsm8k_mathematical_equivalence = Task("scores", "gsm8k_mathematical_equivalence", "GSM8K ⭐", TaskType.MR)
-    gpqa_diamond_en_exact_match = Task("scores", "gpqa_diamond_en_exact_match", "GPQA Diamond EN ⭐", TaskType.HE_EN)
-    gpqa_extended_en_exact_match = Task(
-        "scores", "gpqa_extended_en_exact_match", "GPQA Extended EN ⭐", TaskType.HE_EN
-    )
-    gpqa_main_en_exact_match = Task("scores", "gpqa_main_en_exact_match", "GPQA Main EN ⭐", TaskType.HE_EN)
-    gpqa_diamond_ja_exact_match = Task("scores", "gpqa_diamond_ja_exact_match", "GPQA Diamond JA ⭐", TaskType.HE_JA)
-    gpqa_extended_ja_exact_match = Task(
-        "scores", "gpqa_extended_ja_exact_match", "GPQA Extended JA ⭐", TaskType.HE_JA
-    )
-    gpqa_main_ja_exact_match = Task("scores", "gpqa_main_ja_exact_match", "GPQA Main JA ⭐", TaskType.HE_JA)
-    jamc_qa_exact_match = Task("scores", "jamc-qa_exact_match", "JAMC-QA ⭐", TaskType.QA)
-    jhumaneval_code_exec_sandbox = Task("scores", "jhumaneval_code_exec_sandbox", "JHumanEval ⭐", TaskType.CG)
-    mgsm_mathematical_equivalence = Task("scores", "mgsm_mathematical_equivalence", "MGSM ⭐", TaskType.MR)
-    mmlu_prox_ja_exact_match = Task("scores", "mmlu_prox_ja_exact_match", "MMLU Prox JA ⭐", TaskType.HE_JA)
-    mmlu_prox_en_exact_match = Task("scores", "mmlu_prox_en_exact_match", "MMLU Prox EN ⭐", TaskType.HE_EN)
-    mif_eval_ja_mifeval_strict = Task("scores", "mif_eval_ja_mifeval_strict", "MIF Eval JA ⭐", TaskType.IF)
-    mif_eval_en_mifeval_strict = Task("scores", "mif_eval_en_mifeval_strict", "MIF Eval EN ⭐", TaskType.IF)
-    mmmlu_exact_match = Task("scores", "mmmlu_exact_match", "MMMLU ⭐", TaskType.HE_JA)
-    openbookqa_exact_match = Task("scores", "openbookqa_exact_match", "OpenBookQA ⭐", TaskType.HE_EN)
-    polymath_en_polymath_weighted_accuracy = Task(
-        "scores", "polymath-en_polymath_weighted_accuracy", "Polymath EN ⭐", TaskType.MR
-    )
-    polymath_ja_polymath_weighted_accuracy = Task(
-        "scores", "polymath-ja_polymath_weighted_accuracy", "Polymath JA ⭐", TaskType.MR
-    )
-    triviaqa_triviaqa_f1 = Task("scores", "triviaqa_triviaqa_f1", "TriviaQA ⭐", TaskType.QA)
-    winogrande_xl_exact_match = Task("scores", "winogrande_xl_exact_match", "WinoGrande XL ⭐", TaskType.CR)
-    # HLE/JHLE - Humanity's Last Exam
-    hle_hle_exact_match = Task("scores", "hle_hle_exact_match", "HLE ⭐", TaskType.HE_EN)
-    jhle_hle_exact_match = Task("scores", "jhle_hle_exact_match", "JHLE ⭐", TaskType.HE_JA)
-
-
-# ---------------------------------------------------
-
-# Your leaderboard name
+def _auto_display_name(dataset_name: str) -> str:
+    return dataset_name.replace("_", " ").replace("-", " ").title()
+
+
+def _build_tasks(yaml_config: dict) -> list[Task]:
+    """upstream yaml config から Tasks リストを動的に構築する。"""
+    categories = yaml_config["categories"]
+    tasks: list[Task] = []
+
+    tasks.append(Task("scores", "AVG", "AVG", TaskType.AVG, average=True))
+
+    for cat_name, _cat_def in categories.items():
+        task_type = _CATEGORY_TO_TASK_TYPE.get(cat_name, TaskType.NotTask)
+        tasks.append(Task("scores", cat_name, f"AVG ({cat_name})", task_type, average=True))
+
+    # upstream yaml に未反映の metric override を補完
+    _local_metric_fixes: dict[str, str] = {
+        "jculture-mcq": "set_f1",
+    }
+
+    for cat_name, cat_def in categories.items():
+        task_type = _CATEGORY_TO_TASK_TYPE.get(cat_name, TaskType.NotTask)
+        default_metric = cat_def["default_metric"]
+        metric_overrides = cat_def.get("metrics", {})
+        for ds in cat_def["datasets"]:
+            metric = _local_metric_fixes.get(ds) or metric_overrides.get(ds, default_metric)
+            metric_col = f"{ds}_{metric}"
+            display_name = _DISPLAY_NAME_OVERRIDES.get(ds, _auto_display_name(ds))
+            tasks.append(Task("scores", metric_col, display_name, task_type))
+
+    return tasks
+
+
+Tasks = _build_tasks(load_upstream_eval_config())
+
 TITLE = """<h1 align="center" id="space-title">🇯🇵 Open Japanese LLM Leaderboard V2 🌸<br>オープン日本語LLMリーダーボード V2</h1>"""
 
-# What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
 The __Open Japanese LLM Leaderboard__ by __[LLM-jp](https://llm-jp.nii.ac.jp/en/)__ evaluates
 the performance of Japanese Large Language Models (LLMs) across 14 categories covering more than 71 tasks from
@@ -189,344 +185,73 @@ __「LLM Benchmark」__ ページでは、疑問符 **「?」** はHugging Face
 参加することができます。
 """
 
-# Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = """
 ## How it works
 📈 We evaluate Japanese Large Language Models across 14 categories covering more than 71 tasks leveraging our evaluation tool [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval), a unified framework to evaluate Japanese LLMs on various evaluation tasks.
 
-**NLI (Natural Language Inference)**
-
-* `Jamp`, a Japanese NLI benchmark focused on temporal inference [Source](https://github.com/tomo-ut/temporalNLI_dataset) (License CC BY-SA 4.0)
-
-* `JaNLI`, Japanese Adversarial Natural Language Inference [Source](https://github.com/verypluming/JaNLI) (License CC BY-SA 4.0)
-
-* `JNLI`, Japanese Natural Language Inference (part of JGLUE) [Source](https://github.com/yahoojapan/JGLUE) (License CC BY-SA 4.0)
-
-* `JSeM`, Japanese semantic test suite [Source](https://github.com/DaisukeBekki/JSeM) (License BSD 3-Clause)
-
-* `JSICK`, Japanese Sentences Involving Compositional Knowledge [Source](https://github.com/verypluming/JSICK) (License CC BY-SA 4.0)
-
-**QA (Question Answering)**
-
-* `JEMHopQA`, Japanese Explainable Multi-hop Question Answering [Source](https://github.com/aiishii/JEMHopQA) (License CC BY-SA 4.0)
-
-* `NIILC`, NIILC Question Answering Dataset [Source](https://github.com/mynlp/niilc-qa) (License CC BY-SA 4.0)
-
-* `JAQKET`, Japanese QA dataset on the subject of quizzes [Source](https://www.nlp.ecei.tohoku.ac.jp/projects/jaqket/) (License CC BY-SA 4.0 - Other licenses are required for corporate usage)
-
-* `TriviaQA`, Reading Comprehension Challenge Dataset [Source](https://nlp.cs.washington.edu/triviaqa/) (License Apache-2.0)
-
-* `DROP`, Discrete Reasoning Over Paragraphs [Source](https://allennlp.org/drop) (License CC BY-SA 4.0)
-
-* `JAMC-QA`, Japanese Advanced Medical Comprehension Question Answering [Source](https://huggingface.co/datasets/llm-jp/jamc-qa) (License CC BY-SA 4.0)
-
-**RC (Reading Comprehension)**
-
-* `JSQuAD`, Japanese version of SQuAD (part of JGLUE) [Source](https://github.com/yahoojapan/JGLUE) (License CC BY-SA 4.0)
-
-**CR (Commonsense Reasoning)**
-
-* `JCommonsenseMorality`, Japanese dataset for evaluating commonsense morality understanding [Source](https://github.com/Language-Media-Lab/commonsense-moral-ja) (License MIT License)
-
-* `JCommonsenseQA`, Japanese version of CommonsenseQA [Source](https://github.com/yahoojapan/JGLUE) (License CC BY-SA 4.0)
-
-* `KUCI`, Kyoto University Commonsense Inference dataset [Source](https://github.com/ku-nlp/KUCI (License CC BY-SA 4.0)
-
-* `WinoGrande`, Winogrande Pronoun Disambiguation [Source](https://huggingface.co/datasets/winogrande) (License Apache-2.0)
-
-**EL (Entity Linking)**
-
-* `chABSA`, Aspect-Based Sentiment Analysis dataset [Source](https://github.com/chakki-works/chABSA-dataset) (License CC BY-SA 4.0)
-
-**FA (Fundamental Analysis)**
-
-* `Wikipedia Annotated Corpus`, [Source](https://github.com/ku-nlp/WikipediaAnnotatedCorpus) (License CC BY-SA 4.0)
-
-List of tasks: (Reading Prediction, Named-entity recognition (NER), Dependency Parsing, Predicate-argument structure analysis (PAS), Coreference Resolution)
-
-**MR (Mathematical Reasoning)**
-
-* `MAWPS`, Japanese version of MAWPS (A Math Word Problem Repository) [Source](https://github.com/nlp-waseda/chain-of-thought-ja-dataset) (License Apache-2.0)
-
-* `MGSM`, Japanese part of MGSM (Multilingual Grade School Math Benchmark) [Source](https://huggingface.co/datasets/juletxara/mgsm) (License MIT License)
-
-* `GSM8K`, Grade School Math 8K [Source](https://github.com/openai/grade-school-math) (License MIT License)
-
-* `AIME`, American Invitational Mathematics Examination [Source](https://artofproblemsolving.com/wiki/index.php/AIME_Problems_and_Solutions) (License Public Domain)
-
-* `Polymath`, Multilevel Multimodal Mathematical Reasoning [Source](https://arxiv.org/abs/2407.21046) (License MIT License)
-
-**MT (Machine Translation)**
-
-* `ALT`, Asian Language Treebank (ALT) - Parallel Corpus [Source](https://www2.nict.go.jp/astrec-att/member/mutiyama/ALT/index.html) (License CC BY-SA 4.0)
-
-* `WikiCorpus`, Japanese-English Bilingual Corpus of Wikipedia's articles about the city of Kyoto [Source](https://alaginrc.nict.go.jp/WikiCorpus/) (License CC BY-SA 3.0)
-
-**STS (Semantic Textual Similarity)**
-
-This task is supported by llm-jp-eval, but it is not included in the evaluation score average.
-
-* `JSTS`,  Japanese version of the STS (Semantic Textual Similarity) (part of JGLUE) [Source](https://github.com/yahoojapan/JGLUE) (License CC BY-SA 4.0)
-
-**HE-EN (Human Examination - English)**
-
-* `MMLU`, Measuring Massive Multitask Language Understanding [Source](https://github.com/hendrycks/test) (License MIT License)
-
-* `GPQA`, Graduate-Level Google-Proof Q&A Benchmark [Source](https://github.com/idavidrein/gpqa) (License MIT License)
-
-* `OpenBookQA`, Open Book Question Answering [Source](https://allenai.org/data/open-book-qa) (License Apache-2.0)
-
-* `HLE`, Humanity's Last Exam [Source](https://huggingface.co/datasets/cais/hle) (License MIT License)
-
-**HE-JA (Human Examination - Japanese)**
-
-* `JMMLU`, Japanese Massive Multitask Language Understanding Benchmark [Source](https://github.com/nlp-waseda/JMMLU) (License CC BY-SA 4.0 (3 tasks under the CC BY-NC-ND 4.0 license)
-
-* `MMMLU`, Japanese version of MMLU [Source](https://huggingface.co/datasets/pfnet/mmmlu) (License MIT License)
-
-* `GPQA (JA)`, Japanese translation of GPQA [Source](https://github.com/idavidrein/gpqa) (License MIT License)
-
-* `JHLE`, Japanese Humanity's Last Exam [Source](https://huggingface.co/datasets/llm-jp/jhle) (License MIT License)
-
-**CG (Code Generation)**
-
-* `MBPP`, Japanese version of Mostly Basic Python Problems (MBPP) [Source](https://huggingface.co/datasets/llm-jp/mbpp-ja) (License CC BY-SA 4.0)
-
-* `JHumanEval`, Japanese version of HumanEval [Source](https://huggingface.co/datasets/kogi-jwu/jhumaneval) (License MIT License)
-
-**BBH (BIG-Bench Hard)**
-
-* `BigBenchHard`, Challenging BIG-Bench tasks with chain-of-thought evaluation [Source](https://github.com/suzgunmirac/BIG-Bench-Hard) (License MIT License)
-
-**IF (Instruction Following)**
-
-* `MIF-Eval`, Multilingual Instruction Following Evaluation [Source](https://huggingface.co/datasets/google/MIF-Eval) (License Apache-2.0)
-
-**SUM (Summarization)**
-
-* `XL-Sum`, XL-Sum: Large-Scale Multilingual Abstractive Summarization for 44 Languages [Source](https://github.com/csebuetnlp/xl-sum) (License CC BY-NC-SA 4.0, due to the non-commercial license, this dataset will not be used, unless you specifically agree to the license and terms of use)
-
-
-## Reproducibility
-To reproduce our results, please follow the instructions of the evalution tool, **llm-jp-eval** available in [Japanese](https://github.com/llm-jp/llm-jp-eval/blob/main/README.md) and in [English](https://github.com/llm-jp/llm-jp-eval/blob/main/README_en.md).
-
-## Average Score Calculation
-The calculation of the average score (AVG) includes only the scores of datasets marked with a ⭐.
-
-## Dataset Details
-For comprehensive information about all datasets used in this leaderboard, including detailed descriptions, data sources, preprocessing methods, and the jaster training dataset, please refer to [DATASET.md](https://github.com/llm-jp/llm-jp-eval/blob/main/DATASET.md) in the llm-jp-eval repository.
-
+For more details, see the [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval) documentation.
 """
 
 LLM_BENCHMARKS_TEXT_JA = """
 ## 仕組み
-📈 評価ツール [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval) を活用し、16種類のタスクで日本語の大規模言語モデルを評価します。このツールは、様々な評価タスクで日本語LLMを評価するための統一的なフレームワークです。
-
-**NLI（自然言語推論）**
-
-* `Jamp`、時間推論に焦点を当てた日本語NLIベンチマーク [ソース](https://github.com/tomo-ut/temporalNLI_dataset)（ライセンス CC BY-SA 4.0）
-
-* `JaNLI`、日本語の敵対的推論データセット [ソース](https://github.com/verypluming/JaNLI)（ライセンス CC BY-SA 4.0）
-
-* `JNLI`、日本語自然言語推論（JGLUEの一部）[ソース](https://github.com/yahoojapan/JGLUE)（ライセンス CC BY-SA 4.0）
-
-* `JSeM`、日本語意味論テストセット [ソース](https://github.com/DaisukeBekki/JSeM)（ライセンス BSD 3-Clause）
-
-* `JSICK`、構成的知識を含む日本語文データセット [ソース](https://github.com/verypluming/JSICK)（ライセンス CC BY-SA 4.0）
-
-**QA（質問応答）**
-
-* `JEMHopQA`、日本語の説明可能なマルチホップ質問応答 [ソース](https://github.com/aiishii/JEMHopQA)（ライセンス CC BY-SA 4.0）
-
-* `NIILC`、NIILC質問応答データセット [ソース](https://github.com/mynlp/niilc-qa)（ライセンス CC BY-SA 4.0）
-
-* `JAQKET`、クイズを題材とした日本語QAデータセット [ソース](https://www.nlp.ecei.tohoku.ac.jp/projects/jaqket/)（ライセンス CC BY-SA 4.0 - 企業利用には別途ライセンスが必要）
-
-**RC（読解）**
-
-* `JSQuAD`、SQuADの日本語版（JGLUEの一部）[ソース](https://github.com/yahoojapan/JGLUE)（ライセンス CC BY-SA 4.0）
-
-**MC（選択式質問応答）**
-
-* `JCommonsenseMorality`、常識的な道徳理解を評価する日本語データセット [ソース](https://github.com/Language-Media-Lab/commonsense-moral-ja)（ライセンス MIT License）
-
-* `JCommonsenseQA`、CommonsenseQAの日本語版 [ソース](https://github.com/yahoojapan/JGLUE)（ライセンス CC BY-SA 4.0）
-
-* `KUCI`、京都大学常識推論データセット [ソース](https://github.com/ku-nlp/KUCI)（ライセンス CC BY-SA 4.0）
-
-**EL（エンティティリンキング）**
-
-* `chABSA`、アスペクトベースの感情分析データセット [ソース](https://github.com/chakki-works/chABSA-dataset)（ライセンス CC BY-SA 4.0）
-
-**FA（基礎解析）**
-
-* `Wikipedia Annotated Corpus`、[ソース](https://github.com/ku-nlp/WikipediaAnnotatedCorpus)（ライセンス CC BY-SA 4.0）
-
-タスク一覧：（読解予測、固有表現認識（NER）、依存構造解析、述語項構造解析（PAS）、共参照解析）
-
-**MR（数学的推論）**
-
-* `MAWPS`、MAWPS（A Math Word Problem Repository）の日本語版 [ソース](https://github.com/nlp-waseda/chain-of-thought-ja-dataset)（ライセンス Apache-2.0）
-
-* `MGSM`、MGSM（Multilingual Grade School Math Benchmark）の日本語部分 [ソース](https://huggingface.co/datasets/juletxara/mgsm)（ライセンス MIT License）
-
-**MT（機械翻訳）**
-
-* `ALT`、アジア言語ツリーバンク（ALT） - 並行コーパス [ソース](https://www2.nict.go.jp/astrec-att/member/mutiyama/ALT/index.html)（ライセンス CC BY-SA 4.0）
-
-* `WikiCorpus`、京都市に関するWikipedia記事の日本語-英語対訳コーパス [ソース](https://alaginrc.nict.go.jp/WikiCorpus/)（ライセンス CC BY-SA 3.0）
-
-**STS（意味的テキスト類似度）**
-
-このタスクはllm-jp-evalでサポートされていますが、平均スコア (AVG) の計算には含まれていません。
-
-* `JSTS`、STS（Semantic Textual Similarity）の日本語版（JGLUEの一部）[ソース](https://github.com/yahoojapan/JGLUE)（ライセンス CC BY-SA 4.0）
-
-**HE（試験問題）**
-
-* `MMLU`、大規模マルチタスク言語理解ベンチマーク（英語） [ソース](https://github.com/hendrycks/test)（ライセンス MIT License）
-
-* `JMMLU`、日本語大規模マルチタスク言語理解ベンチマーク [ソース](https://github.com/nlp-waseda/JMMLU)（ライセンス CC BY-SA 4.0（3つのタスクはCC BY-NC-ND 4.0ライセンス）
-
-* `HLE`、Humanity's Last Exam（英語） [ソース](https://huggingface.co/datasets/cais/hle)（ライセンス MIT License）
-
-* `JHLE`、Humanity's Last Exam（日本語） [ソース](https://huggingface.co/datasets/llm-jp/jhle)（ライセンス MIT License）
-
-**CG（コード生成）**
-
-* `MBPP`、Mostly Basic Python Problems（MBPP）の日本語版 [ソース](https://huggingface.co/datasets/llm-jp/mbpp-ja)（ライセンス CC BY-SA 4.0）
-
-**SUM（要約）**
-
-* `XL-Sum`、44言語の大規模多言語抽象型要約データセットの日本語部分 [ソース](https://github.com/csebuetnlp/xl-sum)（ライセンス CC BY-NC-SA 4.0、非商用ライセンスのため、このデータセットは使用しません。ライセンスと利用規約に明確に同意した場合を除きます）
-
-## 再現性
-結果を再現するには、評価ツール **llm-jp-eval** の指示に従ってください。詳細は [日本語](https://github.com/llm-jp/llm-jp-eval/blob/main/README.md) と [英語](https://github.com/llm-jp/llm-jp-eval/blob/main/README_en.md) でご覧いただけます。
-
-## 平均スコアの計算について
-平均スコア (AVG) の計算には、⭐マークのついたスコアのみが含まれます
-
-## データセット詳細
-リーダーボードで使用されている全データセットの包括的な情報（詳細な説明、データソース、前処理方法、jaster訓練データセットなど）については、llm-jp-evalリポジトリの[DATASET.md](https://github.com/llm-jp/llm-jp-eval/blob/main/DATASET.md)をご参照ください。
+📈 評価ツール [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval) を活用し、14カテゴリ・71以上のタスクで日本語大規模言語モデルを評価しています。
 
+詳細は [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval) のドキュメントをご覧ください。
 """
 
-
 EVALUATION_QUEUE_TEXT = """
-## First Steps Before Submitting a Model
-### 1. Ensure Your Model Loads with AutoClasses
-Verify that you can load your model and tokenizer using AutoClasses:
+## Some good practices before submitting a model
+
+### 1) Make sure you can load your model and tokenizer using AutoClasses:
 ```python
 from transformers import AutoConfig, AutoModel, AutoTokenizer
 config = AutoConfig.from_pretrained("your model name", revision=revision)
 model = AutoModel.from_pretrained("your model name", revision=revision)
 tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
 ```
-Note:
-- If this step fails, debug your model before submitting.
-- Ensure your model is public.
-- Models requiring `use_remote_code=True` are not currently supported.
-### 2. Convert Weights to Safetensors
-[Safetensors](https://huggingface.co/docs/safetensors/index) is a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-### 3. Verify Your Model Open License
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-### 4. Complete Your Model Card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-### 5. Select Appropriate Precision
-The "auto" option supports fp16, fp32, and bf16 precisions. If your model uses any other precision format, please select the appropriate option.
-If auto is specified, precision in config.json is automatically selected.
-### 6. Inference-time Options
-Our evaluation system supports various inference-time parameters:
-
-#### Thinking Parameter
-Models that support the `thinking` parameter (e.g., DeepSeek-R1, QwQ) can be evaluated with this feature enabled. When submitting your model, you can specify whether to use the thinking parameter in the submission form.
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
 
-#### Reasoning Parser
-For models that output reasoning processes before final answers, our system includes a `Reasoning Parser` that automatically extracts the final answer from the model's output. This ensures fair evaluation even for models with different output formats.
+### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
 
-**Note**: If your model uses special output formatting or reasoning tokens, please mention this in your model card to ensure proper evaluation.
-
-### Note about large models
-Currently, we officially support models up to 70B parameters. Depending on the model architecture, models larger than 70B may also be evaluated successfully, but we cannot guarantee this will always be the case.
-
-### Evaluation Timeout
-Each evaluation job has a **30-hour timeout limit**. If the inference does not complete within this time, the evaluation will be marked as **failed**. This may occur with very large models or when computational resources are constrained.
+### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
 
+### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card
 """
+
 EVALUATION_QUEUE_TEXT_JA = """
-## モデル提出前の最初のステップ
-### 1. AutoClasses でモデルが読み込めることを確認
-AutoClasses を使用してモデルとトークナイザーを読み込めることを確認してください:
+## モデル提出前の確認事項
+
+### 1) AutoClassesでモデルとトークナイザーが読み込めることを確認:
 ```python
 from transformers import AutoConfig, AutoModel, AutoTokenizer
 config = AutoConfig.from_pretrained("your model name", revision=revision)
 model = AutoModel.from_pretrained("your model name", revision=revision)
 tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
 ```
-注意:
-- この手順が失敗する場合は、提出前にモデルをデバッグしてください。
-- モデルが公開されていることを確認してください。
-- `use_remote_code=True` を必要とするモデルは現時点ではサポートされていません。
-
-### 2. 重みを Safetensors に変換
-[Safetensors](https://huggingface.co/docs/safetensors/index) は、より安全で高速に読み込めるウェイトの新しい保存形式です。これにより、`Extended Viewer` にモデルのパラメータ数を追加することも可能になります！
-
-### 3. モデルのオープンライセンスを確認
-これはオープン LLM のリーダーボードです。できるだけ多くの人があなたのモデルを使用できることを知ってもらえると嬉しいです🤗
-
-### 4. モデルカードを完成させる
-リーダーボードにモデルの追加情報を掲載する際は、モデルカードから自動的に情報が取得されます
+このステップが失敗する場合は、エラーメッセージに従ってモデルをデバッグしてください。
 
-### 5. 適切なPrecisionの選択
-"auto"オプションはfp16、fp32、bf16のprecisionに対応しています。これら以外のprecisionを使用している場合は、適切なオプションを選択してください。
-また、autoを指定した場合、config.jsonのprecisionが自動的に選択されます。
+### 2) モデルの重みを[safetensors](https://huggingface.co/docs/safetensors/index)に変換
+安全で高速に読み込める新しいフォーマットです。
 
-### 6. 推論時のオプション
-以下の推論時パラメータをサポートしています：
+### 3) モデルにオープンライセンスがあることを確認
+オープンLLMのリーダーボードです。できるだけ多くの人がモデルを使えるようにしましょう 🤗
 
-#### Thinking Parameter
-`thinking`パラメータをサポートするモデル（例：DeepSeek-R1、QwQ）は、この機能を有効にして評価できます。モデル提出時に、提出フォームでthinkingパラメータの使用有無を指定できます。
-Reasoning Parserを対応するモデルのものに変更してください。
-
-### 大規模モデルに関する注意
-現在、70Bパラメータまでのモデルを公式にサポートしています。モデルのアーキテクチャによっては70Bを超えるモデルでも評価できる場合がありますが、必ずしも動作を保証するものではありませんのでご了承ください。
-
-### 評価のタイムアウト
-各評価ジョブには**30時間のタイムアウト制限**が設定されています。この時間内に推論が完了しない場合、評価は**failed**（失敗）としてマークされます。これは非常に大規模なモデルや計算リソースが制約されている場合に発生することがあります。
-
-"""
-
-BOTTOM_LOGO = """
-<div style="display: flex; flex-direction: row; justify-content: center; align-items: center;">
-  <a href="https://llm-jp.nii.ac.jp/en/" style="margin: 0 10px;">
-    <img src="https://raw.githubusercontent.com/AkimfromParis/akimfromparis/refs/heads/main/images/LLM-jp-Logo-Oct-2024.png" alt="LLM-jp" style="max-height: 100px;">
-  </a>
-  <a href="https://mdx.jp/" style="margin: 0 10px;">
-    <img src="https://raw.githubusercontent.com/AkimfromParis/akimfromparis/refs/heads/main/images/MDX-Logo-Oct-2024.jpg" alt="MDX" style="max-height: 100px;">
-  </a>
-  <a href="https://huggingface.co/" style="margin: 0 10px;">
-    <img src="https://raw.githubusercontent.com/AkimfromParis/akimfromparis/refs/heads/main/images/HuggingFace-Logo-Oct-2024.png" alt="HuggingFace" style="max-height: 100px;">
-  </a>
-</div>
+### 4) モデルカードを記入
+リーダーボードに追加情報が表示される際、モデルカードから自動的に取得されます。
 """
 
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_LABEL_JA = "引用の際は、次のスニペットをコピーしてご利用ください"
-
-CITATION_BUTTON_TEXT = r"""@misc{OJLL,
-  author = {Miyao, Yusuke and Ishida, Shigeki and Okamoto, Takumi and Han, Namgi and Mousterou, Akim and Fourrier, Clémentine and Hayashi, Toshihiro and Tachibana, Yuichiro},
-  title = {Open Japanese LLM Leaderboard},
-  year = {2024},
-  publisher = {OJLL},
-  howpublished = "\url{https://huggingface.co/spaces/llm-jp/open-japanese-llm-leaderboard}"
-}
-@misc{llmjp2024llmjpcrossorganizationalprojectresearch,
-      title={LLM-jp: A Cross-organizational Project for the Research and Development of Fully Open Japanese LLMs},
-      author={LLM-jp and : and Akiko Aizawa and Eiji Aramaki and Bowen Chen and Fei Cheng and Hiroyuki Deguchi and Rintaro Enomoto and Kazuki Fujii and Kensuke Fukumoto and Takuya Fukushima and Namgi Han and Yuto Harada and Chikara Hashimoto and Tatsuya Hiraoka and Shohei Hisada and Sosuke Hosokawa and Lu Jie and Keisuke Kamata and Teruhito Kanazawa and Hiroki Kanezashi and Hiroshi Kataoka and Satoru Katsumata and Daisuke Kawahara and Seiya Kawano and Atsushi Keyaki and Keisuke Kiryu and Hirokazu Kiyomaru and Takashi Kodama and Takahiro Kubo and Yohei Kuga and Ryoma Kumon and Shuhei Kurita and Sadao Kurohashi and Conglong Li and Taiki Maekawa and Hiroshi Matsuda and Yusuke Miyao and Kentaro Mizuki and Sakae Mizuki and Yugo Murawaki and Ryo Nakamura and Taishi Nakamura and Kouta Nakayama and Tomoka Nakazato and Takuro Niitsuma and Jiro Nishitoba and Yusuke Oda and Hayato Ogawa and Takumi Okamoto and Naoaki Okazaki and Yohei Oseki and Shintaro Ozaki and Koki Ryu and Rafal Rzepka and Keisuke Sakaguchi and Shota Sasaki and Satoshi Sekine and Kohei Suda and Saku Sugawara and Issa Sugiura and Hiroaki Sugiyama and Hisami Suzuki and Jun Suzuki and Toyotaro Suzumura and Kensuke Tachibana and Yu Takagi and Kyosuke Takami and Koichi Takeda and Masashi Takeshita and Masahiro Tanaka and Kenjiro Taura and Arseny Tolmachev and Nobuhiro Ueda and Zhen Wan and Shuntaro Yada and Sakiko Yahata and Yuya Yamamoto and Yusuke Yamauchi and Hitomi Yanaka and Rio Yokota and Koichiro Yoshino},
-      year={2024},
-      eprint={2407.03963},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL},
-      url={https://arxiv.org/abs/2407.03963},
+CITATION_BUTTON_LABEL = "Citation"
+CITATION_BUTTON_LABEL_JA = "引用"
+CITATION_BUTTON_TEXT = r"""
+@misc{open-japanese-llm-leaderboard-v2,
+    author = {LLM-jp},
+    title = {Open Japanese LLM Leaderboard V2},
+    year = {2025},
+    howpublished = {\url{https://huggingface.co/spaces/llm-jp/open-japanese-llm-leaderboard-v2}}
 }
 """
+
+BOTTOM_LOGO = ""

src/display/utils.py

@@ -10,9 +10,6 @@ def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
 @dataclass(frozen=True)
 class ColumnContent:
     name: str
@@ -25,26 +22,24 @@ class ColumnContent:
     average: bool = False
 
 
-## Leaderboard columns
 auto_eval_column_dict = []
-# Init
+
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-# Scores
-# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
+    # task.metric をフィールド名に使う (ハイフンはアンダースコアに変換)
+    field_name = task.metric.replace("-", "_")
     auto_eval_column_dict.append([
-        task.name,
+        field_name,
         ColumnContent,
         ColumnContent(
-            task.value.col_name,
+            task.col_name,
             "number",
-            displayed_by_default=(task.value.task_type == TaskType.AVG or task.value.average),
-            task_type=task.value.task_type,
-            average=task.value.average,
+            displayed_by_default=(task.task_type == TaskType.AVG or task.average),
+            task_type=task.task_type,
+            average=task.average,
         ),
     ])
-# Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
@@ -61,12 +56,10 @@ auto_eval_column_dict.append(["co2_emission", ColumnContent, ColumnContent("CO2
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 auto_eval_column_dict.append(["row_id", ColumnContent, ColumnContent("ID", "number", False, dummy=True)])
 
-# We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict)
 
 
-## For the queue columns in the submission tab
-class EvalQueueColumn:  # Queue column (not a dataclass - used as class attributes)
+class EvalQueueColumn:
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
     model_type = ColumnContent("model_type", "str", True)
@@ -83,21 +76,6 @@ class EvalQueueColumn:  # Queue column (not a dataclass - used as class attribut
     reasoning_parser = ColumnContent("reasoning_parser", "str", False)
 
 
-# This class is used to store the model data in the queue
-@dataclass(frozen=True)
-class EvalQueuedModel:
-    model: str
-    revision: str
-    precision: str
-    add_special_tokens: str
-    llm_jp_eval_version: str
-    vllm_version: str
-    apply_chat_template: bool = False
-    enable_thinking: bool = False
-    reasoning_parser: str = ""
-
-
-## All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
@@ -173,7 +151,7 @@ class ApplyChatTemplate(Enum):
 
 
 class LLMJpEvalVersion(Enum):
-    current = ModelDetails("v2.0.0")
+    current = ModelDetails("v2.1.4")
 
     @classmethod
     def from_str(cls, version: str) -> "LLMJpEvalVersion":
@@ -192,14 +170,13 @@ class VllmVersion(Enum):
         raise ValueError(f"Unsupported VLLM version: {version}")
 
 
-# Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn)]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+BENCHMARK_COLS = [t.col_name for t in Tasks]
 
 NUMERIC_INTERVALS = {
     "0~3B": pd.Interval(0, 3, closed="right"),

src/envs.py

@@ -20,4 +20,11 @@ CACHE_PATH = pathlib.Path(os.getenv("HF_HOME", "."))
 # Local caches
 EVAL_REQUESTS_PATH = CACHE_PATH / "eval-queue"
 
+# llm-jp-eval upstream の dataset 定義 (dev branch を直接参照することで
+# 新 dataset が upstream にマージされた瞬間にフロントが認識する)
+LLM_JP_EVAL_DATASETS_URL = (
+    "https://raw.githubusercontent.com/llm-jp/llm-jp-eval/"
+    "dev/eval_configs/all_datasets.yaml"
+)
+
 API = HfApi(token=HF_TOKEN)

src/populate.py

@@ -1,16 +1,14 @@
 import json
 import os
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 
 import pandas as pd
 from huggingface_hub import hf_hub_download
 
 from src.about import Tasks
-from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 
-# The values of these columns are in the range of 0-100
-# We normalize them to 0-1
 COLUMNS_TO_NORMALIZE = [
     "ALT E to J BLEU",
     "ALT J to E BLEU",
@@ -23,18 +21,22 @@ COLUMNS_TO_NORMALIZE = [
 ]
 
 
-def get_leaderboard_df(contents_repo: str, cols: list[str], benchmark_cols: list[str]) -> pd.DataFrame:
+def get_raw_leaderboard_df(contents_repo: str) -> pd.DataFrame:
+    """rename / 正規化を行わない素の leaderboard.parquet を返す。"""
     parquet_path = hf_hub_download(
         repo_id=contents_repo,
         filename="leaderboard.parquet",
         repo_type="dataset",
     )
-    df = pd.read_parquet(parquet_path, engine="pyarrow")
+    return pd.read_parquet(parquet_path, engine="pyarrow")
+
+
+def get_leaderboard_df(contents_repo: str, cols: list[str], benchmark_cols: list[str]) -> pd.DataFrame:  # noqa: ARG001
+    df = get_raw_leaderboard_df(contents_repo)
     df["Model"] = df["model"].map(make_clickable_model)
     df["T"] = df["model_type"].map(lambda x: x.split(":")[0].strip())
-    df = df.rename(columns={task.value.metric: task.value.col_name for task in Tasks})
+    df = df.rename(columns={task.metric: task.col_name for task in Tasks})
 
-    # Rename columns only if they exist
     rename_dict = {
         "architecture": "Architecture",
         "weight_type": "Weight type",
@@ -53,11 +55,9 @@ def get_leaderboard_df(contents_repo: str, cols: list[str], benchmark_cols: list
         "inference_time_seconds": "Inference Time (s)",
         "co2_emission_kg": "CO2 (kg)",
     }
-    # Only rename columns that exist in the dataframe
     rename_dict = {k: v for k, v in rename_dict.items() if k in df.columns}
     df = df.rename(columns=rename_dict)
 
-    # Normalize bool columns to string "True"/"False" for consistent filtering
     bool_columns = ["Add Special Tokens", "Enable Thinking", "Apply Chat Template"]
     for col in bool_columns:
         if col in df.columns:
@@ -65,22 +65,16 @@ def get_leaderboard_df(contents_repo: str, cols: list[str], benchmark_cols: list
         else:
             df[col] = "False"
 
-    # Add a row ID column
     df[AutoEvalColumn.row_id.name] = range(len(df))
 
-    # Normalize the columns
     available_columns_to_normalize = [col for col in COLUMNS_TO_NORMALIZE if col in df.columns]
     df[available_columns_to_normalize] = df[available_columns_to_normalize] / 100
 
     df = df.sort_values(by=[AutoEvalColumn.AVG.name], ascending=False)
 
-    # Only select columns that exist
     available_cols = [col for col in cols if col in df.columns]
     df = df[available_cols].round(decimals=4)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-
     return df
 
 
@@ -97,7 +91,7 @@ def _compute_elapsed_time(time_str: str | None) -> str:
         return "-"
     try:
         submitted = datetime.fromisoformat(time_str.replace("Z", "+00:00"))
-        delta = datetime.now(timezone.utc) - submitted
+        delta = datetime.now(UTC) - submitted
         total_seconds = int(delta.total_seconds())
         if total_seconds < 0:
             return "-"
@@ -176,12 +170,10 @@ def get_evaluation_queue_df(save_path: str, cols: list[str]) -> list[pd.DataFram
     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
     failed_list = [e for e in all_evals if e["status"] == "FAILED"]
 
-    # Add queue position to pending list (sorted by submitted_time)
     pending_list = sorted(pending_list, key=lambda x: x.get("submitted_time", ""))
     for i, entry in enumerate(pending_list):
         entry["queue_position"] = i + 1
 
-    # Running models: position 0 means currently being processed
     for entry in running_list:
         entry["queue_position"] = 0
 

src/submission/check_validity.py

@@ -1,15 +1,96 @@
 import json
 import os
-import pathlib
 
 import huggingface_hub
+import pandas as pd
 import requests
 from huggingface_hub import ModelCard
 from huggingface_hub.hf_api import ModelInfo
 from transformers import AutoConfig
 from transformers.models.auto.tokenization_auto import AutoTokenizer
 
-from src.display.utils import EvalQueuedModel
+# parquet 上のメタ列 (どの dataset の score でもない)
+_LEADERBOARD_META_COLS = frozenset({
+    "model_type", "model", "revision", "add_special_tokens",
+    "llm_jp_eval_version", "vllm_version", "precision",
+    "architecture", "license", "params", "likes", "num_few_shot",
+    "apply_chat_template", "enable_thinking", "reasoning_parser",
+    "inference_time_seconds", "co2_emission_kg",
+})
+
+# parquet 上の集計列 (個別 dataset 由来ではなく category 平均)
+_LEADERBOARD_AGGREGATE_COLS = frozenset({
+    "NLI", "QA", "RC", "CR", "HE-JA", "HE-EN", "EL", "FA",
+    "MR", "MT", "CG", "SUM", "IF", "BBH", "LM", "AVG",
+})
+
+
+def build_dataset_column_map(
+    upstream_datasets: frozenset[str] | set[str],
+    parquet_columns: list[str],
+) -> dict[str, list[str]]:
+    """upstream dataset 名と parquet 列名を longest-prefix-match で結びつけ、
+    {dataset_name: [属する列名 ...]} の dict を返す。
+
+    parquet 列は `{dataset}_{metric}` パターンで 1 つの dataset に対し複数列が
+    存在する (例: aio_ool, aio_char_f1, aio_exact_match)。各列は最も長く
+    マッチする dataset に割り当てられるので、`gpqa_diamond_en` と
+    `gpqa_diamond_ja` のような兄弟も衝突しない。
+    """
+    sorted_datasets = sorted(upstream_datasets, key=len, reverse=True)
+    mapping: dict[str, list[str]] = {d: [] for d in upstream_datasets}
+    for col in parquet_columns:
+        if col in _LEADERBOARD_META_COLS or col in _LEADERBOARD_AGGREGATE_COLS:
+            continue
+        for dataset in sorted_datasets:
+            if col == dataset or col.startswith(f"{dataset}_"):
+                mapping[dataset].append(col)
+                break
+    return mapping
+
+
+def compute_already_evaluated_datasets(
+    df: pd.DataFrame,
+    upstream_datasets: frozenset[str] | set[str],
+    *,
+    model: str,
+    revision: str,
+    precision: str,
+    add_special_tokens: str,
+    apply_chat_template: bool,
+    enable_thinking: bool,
+) -> set[str]:
+    """parquet から指定モデル設定に該当する行を抽出し、
+    各 upstream dataset について「対応する列のいずれかが non-null」なら
+    評価済みと判定し、その集合を返す。
+    """
+    if df.empty:
+        return set()
+
+    apply_ct_col = df["apply_chat_template"].where(df["apply_chat_template"].notna(), False)
+    enable_th_col = df["enable_thinking"].where(df["enable_thinking"].notna(), False)
+    cond = (
+        (df["model"] == model)
+        & (df["revision"] == revision)
+        & (df["precision"] == precision)
+        & (df["add_special_tokens"] == add_special_tokens)
+        & (apply_ct_col == apply_chat_template)
+        & (enable_th_col == enable_thinking)
+    )
+
+    matched = df[cond]
+    if matched.empty:
+        return set()
+
+    column_map = build_dataset_column_map(upstream_datasets, list(df.columns))
+    evaluated: set[str] = set()
+    for dataset, cols in column_map.items():
+        if not cols:
+            # parquet にまだ存在しない dataset (= upstream で新規追加されたばかり)
+            continue
+        if matched[cols].notna().any().any():
+            evaluated.add(dataset)
+    return evaluated
 
 
 def check_model_card(repo_id: str) -> tuple[bool, str]:
@@ -90,24 +171,3 @@ def get_model_arch(model_info: ModelInfo):
     """Gets the model architecture from the configuration"""
     return model_info.config.get("architectures", "Unknown")
 
-
-def already_submitted_models(requested_models_dir: pathlib.Path) -> set[EvalQueuedModel]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    queued_models = set()
-    for json_path in requested_models_dir.glob("*/*.json"):
-        with json_path.open() as f:
-            info = json.load(f)
-        queued_models.add(
-            EvalQueuedModel(
-                model=info["model"],
-                revision=info["revision"],
-                precision=info["precision"],
-                add_special_tokens=info["add_special_tokens"],
-                llm_jp_eval_version=info["llm_jp_eval_version"],
-                vllm_version=info["vllm_version"],
-                apply_chat_template=info["apply_chat_template"],
-                enable_thinking=info["enable_thinking"],
-                reasoning_parser=info["reasoning_parser"],
-            )
-        )
-    return queued_models

src/submission/submit.py

@@ -1,21 +1,54 @@
 import json
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 
 import gradio as gr
 import torch
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.display.utils import EvalQueuedModel, LLMJpEvalVersion, VllmVersion
-from src.envs import API, EVAL_REQUESTS_PATH, HF_TOKEN, QUEUE_REPO
-from src.submission.check_validity import already_submitted_models, check_model_card, is_model_on_hub
-
-REQUESTED_MODELS: set[EvalQueuedModel] = set()
+from src.display.utils import LLMJpEvalVersion, VllmVersion
+from src.envs import API, CONTENTS_REPO, EVAL_REQUESTS_PATH, HF_TOKEN, QUEUE_REPO
+from src.populate import get_raw_leaderboard_df
+from src.submission.check_validity import (
+    check_model_card,
+    compute_already_evaluated_datasets,
+    is_model_on_hub,
+    load_canonical_dataset_set,
+)
+from src.upstream import load_upstream_eval_config
 
 LLM_JP_EVAL_VERSION = LLMJpEvalVersion.current.value.name
 VLLM_VERSION = VllmVersion.current.value.name
 
 
-def add_new_eval(  # noqa: C901
+def _format_datasets_by_category(datasets: list[str]) -> str:
+    """datasets をカテゴリ別に折りたたみ (HTML details) で整形する。"""
+    config = load_upstream_eval_config()
+    categories = config.get("categories", {})
+
+    ds_set = set(datasets)
+    categorized: dict[str, list[str]] = {}
+    assigned: set[str] = set()
+
+    for cat_name, cat_def in categories.items():
+        desc = cat_def.get("description", cat_name)
+        matched = [d for d in cat_def["datasets"] if d in ds_set]
+        if matched:
+            categorized[f"{cat_name} - {desc}"] = matched
+            assigned.update(matched)
+
+    uncategorized = [d for d in datasets if d not in assigned]
+    if uncategorized:
+        categorized["Other"] = uncategorized
+
+    lines: list[str] = []
+    for cat_label, ds_list in categorized.items():
+        items = " ".join(f"`{d}`" for d in ds_list)
+        lines.append(f"<details><summary><b>{cat_label}</b> ({len(ds_list)})</summary>\n\n{items}\n\n</details>")
+
+    return "\n".join(lines)
+
+
+def _validate_and_resolve(  # noqa: C901
     model_id: str,
     revision: str,
     precision: str,
@@ -24,40 +57,17 @@ def add_new_eval(  # noqa: C901
     apply_chat_template: str,
     enable_thinking: str,
     reasoning_parser: str,
-    profile: gr.OAuthProfile | None = None,
-) -> str:
-    """Add a new model evaluation request
-
-    Args:
-        model_id: HuggingFace model ID
-        revision: Git revision (branch/tag/commit)
-        precision: Model precision (float16/bfloat16/float32/auto)
-        model_type: Model type (pretrained/fine-tuned/etc)
-        add_special_tokens: Whether to add special tokens
-        apply_chat_template: Whether to apply chat template
-        enable_thinking: Whether to enable thinking mode
-        reasoning_parser: Reasoning parser type (qwen3, etc)
-        profile: User's OAuth profile (required for authentication)
-
-    Returns:
-        Status message (success/error)
-    """
-    global REQUESTED_MODELS
-
-    # Check OAuth authentication
+    profile: gr.OAuthProfile | None,
+) -> dict | str:
+    """Validate inputs and compute differential datasets. Returns eval_entry dict on success, error string on failure."""
     if profile is None:
         return styled_error("Please log in with your Hugging Face account to submit a model.")
 
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS = already_submitted_models(EVAL_REQUESTS_PATH)
-
     revision = revision or "main"
 
-    # Convert string to boolean
     apply_chat_template_bool = apply_chat_template == "True"
     enable_thinking_bool = enable_thinking == "True"
 
-    # Validate configuration dependencies
     if enable_thinking_bool:
         if not apply_chat_template_bool:
             return styled_error("Enable Thinking requires Apply Chat Template to be True.")
@@ -66,7 +76,6 @@ def add_new_eval(  # noqa: C901
     elif reasoning_parser and reasoning_parser.strip():
         return styled_error("Reasoning Parser requires Enable Thinking to be True.")
 
-    # Is the model on the hub?
     model_on_hub, error, config = is_model_on_hub(
         model_name=model_id, revision=revision, token=HF_TOKEN, test_tokenizer=True
     )
@@ -87,38 +96,14 @@ def add_new_eval(  # noqa: C901
                 "Unable to retrieve a valid dtype from config.json. Please select an appropriate one from fp16/fp32/bf16 and resubmit."
             )
 
-    model_data = EvalQueuedModel(
-        model=model_id,
-        revision=revision,
-        precision=precision,
-        add_special_tokens=add_special_tokens,
-        llm_jp_eval_version=LLM_JP_EVAL_VERSION,
-        vllm_version=VLLM_VERSION,
-        apply_chat_template=apply_chat_template_bool,
-        enable_thinking=enable_thinking_bool,
-        reasoning_parser=reasoning_parser,
-    )
-
-    if model_data in REQUESTED_MODELS:
-        return styled_warning("This model has already been submitted with the same configuration.")
-
-    if "/" in model_id:
-        user_or_org, model_name = model_id.split("/")
-    else:
-        user_or_org, model_name = "", model_id
-
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
 
-    # Is the model info correctly filled?
     try:
         model_info = API.model_info(repo_id=model_id, revision=revision)
     except Exception:
         return styled_error("Could not get your model information. Please fill it up properly.")
 
-    # Were the model card and license filled?
     try:
         _ = model_info.cardData["license"]
     except Exception:
@@ -128,48 +113,146 @@ def add_new_eval(  # noqa: C901
     if not modelcard_ok:
         return styled_error(error_msg)
 
-    # Seems good, creating the eval
-    print("Adding new eval")
+    try:
+        canonical_datasets = load_canonical_dataset_set()
+    except Exception as e:
+        return styled_error(f"Failed to fetch upstream dataset list: {e}")
+
+    try:
+        leaderboard_df = get_raw_leaderboard_df(CONTENTS_REPO)
+    except Exception as e:
+        return styled_error(f"Failed to load leaderboard.parquet: {e}")
+
+    already_evaluated = compute_already_evaluated_datasets(
+        leaderboard_df,
+        canonical_datasets,
+        model=model_id,
+        revision=revision,
+        precision=precision,
+        add_special_tokens=add_special_tokens,
+        apply_chat_template=apply_chat_template_bool,
+        enable_thinking=enable_thinking_bool,
+    )
+
+    datasets_to_run = sorted(canonical_datasets - already_evaluated)
+
+    if not datasets_to_run:
+        return styled_warning(
+            "All datasets in the current evaluation set have already been "
+            "evaluated for this model configuration."
+        )
 
-    eval_entry = {
+    if "/" in model_id:
+        user_or_org, model_name = model_id.split("/")
+    else:
+        user_or_org, model_name = "", model_id
+
+    return {
         "model_type": model_type,
         "model": model_id,
         "precision": precision,
         "revision": revision,
         "add_special_tokens": add_special_tokens,
-        "num_few_shot": 4,  # Default few-shot count
+        "num_few_shot": 4,
         "llm_jp_eval_version": LLM_JP_EVAL_VERSION,
         "vllm_version": VLLM_VERSION,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "submitted_by": profile.username,
         "apply_chat_template": apply_chat_template_bool,
         "enable_thinking": enable_thinking_bool,
         "reasoning_parser": reasoning_parser if enable_thinking_bool else "",
+        "datasets": datasets_to_run,
+        "_user_or_org": user_or_org,
+        "_model_name": model_name,
+        "_already_evaluated_count": len(already_evaluated),
+        "_canonical_count": len(canonical_datasets),
     }
 
-    print("Creating eval file")
+
+def preview_eval(
+    model_id: str,
+    revision: str,
+    precision: str,
+    model_type: str,
+    add_special_tokens: str,
+    apply_chat_template: str,
+    enable_thinking: str,
+    reasoning_parser: str,
+    profile: gr.OAuthProfile | None = None,
+) -> tuple:
+    """Validate and show confirmation modal with dataset list.
+
+    Returns: (status_text, modal_content, modal_visible, eval_entry_state)
+    """
+    result = _validate_and_resolve(
+        model_id, revision, precision, model_type,
+        add_special_tokens, apply_chat_template,
+        enable_thinking, reasoning_parser, profile,
+    )
+
+    if isinstance(result, str):
+        return result, "", gr.update(visible=False), None
+
+    datasets_to_run = result["datasets"]
+    already_count = result["_already_evaluated_count"]
+    canonical_count = result["_canonical_count"]
+
+    if already_count > 0:
+        header = f"### 差分評価: {len(datasets_to_run)} / {canonical_count} datasets\n{already_count} datasets は既存スコアを流用\n\n"
+    else:
+        header = f"### フル評価: {len(datasets_to_run)} datasets\n\n"
+
+    modal_body = header + _format_datasets_by_category(datasets_to_run)
+
+    return "", modal_body, gr.update(visible=True), result
+
+
+def confirm_eval(
+    eval_entry_state: dict | None,
+    profile: gr.OAuthProfile | None = None,
+) -> tuple:
+    """Submit the eval request to queue.
+
+    Returns: (status_text, modal_visible, state_cleared)
+    """
+    if eval_entry_state is None:
+        return styled_error("No pending submission. Please click Submit first."), gr.update(visible=False), None
+
+    if profile is None:
+        return styled_error("Please log in with your Hugging Face account."), gr.update(visible=False), None
+
+    entry = {k: v for k, v in eval_entry_state.items() if not k.startswith("_")}
+    entry["status"] = "PENDING"
+    entry["submitted_time"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+    entry["submitted_by"] = profile.username
+
+    user_or_org = eval_entry_state["_user_or_org"]
+    model_name = eval_entry_state["_model_name"]
+
     out_dir = EVAL_REQUESTS_PATH / user_or_org
     out_dir.mkdir(parents=True, exist_ok=True)
-    out_file_name = f"{model_name}_{current_time.replace(':','-')}.json"
+    out_file_name = f"{model_name}_{entry['submitted_time'].replace(':', '-')}.json"
     out_path = out_dir / out_file_name
 
     with out_path.open("w") as f:
-        f.write(json.dumps(eval_entry))
+        f.write(json.dumps(entry))
 
-    print("Uploading eval file")
     API.upload_file(
         path_or_fileobj=out_path,
         path_in_repo=out_path.relative_to(EVAL_REQUESTS_PATH).as_posix(),
         repo_id=QUEUE_REPO,
         repo_type="dataset",
-        commit_message=f"Add {model_id} to eval queue",
+        commit_message=f"Add {entry['model']} to eval queue",
     )
-    REQUESTED_MODELS.add(model_data)
-
-    # Remove the local file
     out_path.unlink()
 
     return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )
+        "Your request has been submitted to the evaluation queue!\n"
+        "Please wait for up to an hour for the model to show in the PENDING list."
+    ), gr.update(visible=False), None
+
+
+def cancel_eval() -> tuple:
+    """Cancel the pending submission.
+
+    Returns: (status_text, modal_visible, state_cleared)
+    """
+    return "Submission cancelled.", gr.update(visible=False), None

src/upstream.py

@@ -0,0 +1,25 @@
+"""llm-jp-eval upstream から all_datasets.yaml を取得するモジュール。"""
+
+import functools
+
+import requests
+import yaml
+
+from src.envs import LLM_JP_EVAL_DATASETS_URL
+
+
+@functools.lru_cache(maxsize=1)
+def load_upstream_eval_config() -> dict:
+    """llm-jp-eval upstream から all_datasets.yaml の全内容を取得して返す。
+
+    categories / datasets / dataset_info_overrides を含む dict。
+    プロセス起動中は lru_cache で memoize する。
+    """
+    response = requests.get(LLM_JP_EVAL_DATASETS_URL, timeout=10)
+    response.raise_for_status()
+    return yaml.safe_load(response.text)
+
+
+def load_canonical_dataset_set() -> frozenset[str]:
+    """upstream yaml の `datasets:` フィールドを frozenset として返す。"""
+    return frozenset(load_upstream_eval_config()["datasets"])

style.css

@@ -141,3 +141,36 @@
 #llm-benchmark-tab-table table th:last-child {
   display: none;
 }
+
+/* Confirmation modal overlay */
+.confirm-modal-overlay {
+  position: fixed !important;
+  top: 0 !important;
+  left: 0 !important;
+  width: 100vw !important;
+  height: 100vh !important;
+  max-height: 100vh !important;
+  background: rgba(0, 0, 0, 0.5) !important;
+  z-index: 9999 !important;
+  display: flex !important;
+  align-items: center !important;
+  justify-content: center !important;
+  padding: 0 !important;
+  margin: 0 !important;
+  gap: 0 !important;
+  border: none !important;
+  flex-grow: 0 !important;
+}
+
+.confirm-modal-box {
+  background: var(--background-fill-primary);
+  border: 1px solid var(--border-color-primary) !important;
+  border-radius: 12px;
+  padding: 24px !important;
+  max-width: 600px;
+  width: 90%;
+  max-height: 70vh;
+  overflow-y: auto;
+  box-shadow: 0 8px 32px rgba(0, 0, 0, 0.3);
+  flex-grow: 0 !important;
+}