add chat template option

app.py

@@ -29,6 +29,7 @@ from src.display.utils import (
     NUMERIC_INTERVALS,
     TYPES,
     AddSpecialTokens,
+    ApplyChatTemplate,
     AutoEvalColumn,
     EnableThinking,
     LLMJpEvalVersion,
@@ -100,6 +101,7 @@ def filter_models(
     version_query: list[str],
     vllm_query: list[str],
     enable_thinking_query: list[str],
+    apply_chat_template_query: list[str],
 ) -> pd.DataFrame:
     # Filter by model type
     type_emoji = [t.split()[0] for t in type_query]
@@ -134,6 +136,9 @@ def filter_models(
     # Filter by enable_thinking
     df = df[df["Enable Thinking"].isin(enable_thinking_query)]
 
+    # Filter by apply_chat_template
+    df = df[df["Apply Chat Template"].isin(apply_chat_template_query)]
+
     return df
 
 
@@ -183,6 +188,7 @@ def update_table(
     version_query: list[str],
     vllm_query: list[str],
     enable_thinking_query: list[str],
+    apply_chat_template_query: list[str],
     query: str,
     *columns,
 ) -> pd.DataFrame:
@@ -197,6 +203,7 @@ def update_table(
         version_query,
         vllm_query,
         enable_thinking_query,
+        apply_chat_template_query,
     )
     df = search_models_by_multiple_names(df, query)
     df = select_columns(df, columns)
@@ -221,6 +228,7 @@ if len(leaderboard_df) > 0:
         [i.value.name for i in LLMJpEvalVersion],
         [i.value.name for i in VllmVersion],
         [i.value.name for i in EnableThinking],
+        [i.value.name for i in ApplyChatTemplate],
     )
     leaderboard_df = select_columns(leaderboard_df, INITIAL_COLUMNS)
 else:
@@ -445,6 +453,12 @@ with gr.Blocks() as demo_leaderboard:
                 value=[i.value.name for i in EnableThinking],
                 elem_id="filter-columns-enable-thinking",
             )
+            filter_columns_apply_chat_template = gr.CheckboxGroup(
+                label="Apply Chat Template",
+                choices=[i.value.name for i in ApplyChatTemplate],
+                value=[i.value.name for i in ApplyChatTemplate],
+                elem_id="filter-columns-apply-chat-template",
+            )
 
     leaderboard_table = gr.Dataframe(
         value=leaderboard_df,
@@ -487,6 +501,7 @@ with gr.Blocks() as demo_leaderboard:
             filter_columns_version.change,
             filter_columns_vllm.change,
             filter_columns_enable_thinking.change,
+            filter_columns_apply_chat_template.change,
             search_bar.submit,
         ]
         + [shown_columns.change for shown_columns in shown_columns_dict.values()],
@@ -500,6 +515,7 @@ with gr.Blocks() as demo_leaderboard:
             filter_columns_version,
             filter_columns_vllm,
             filter_columns_enable_thinking,
+            filter_columns_apply_chat_template,
             search_bar,
         ]
         + list(shown_columns_dict.values()),

src/display/utils.py

@@ -62,6 +62,7 @@ auto_eval_column_dict.append(
 )
 auto_eval_column_dict.append(["vllm_version", ColumnContent, ColumnContent("vllm version", "str", False)])
 auto_eval_column_dict.append(["enable_thinking", ColumnContent, ColumnContent("Enable Thinking", "bool", False)])
+auto_eval_column_dict.append(["apply_chat_template", ColumnContent, ColumnContent("Apply Chat Template", "bool", False)])
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 auto_eval_column_dict.append(["row_id", ColumnContent, ColumnContent("ID", "number", False, dummy=True)])
 
@@ -170,6 +171,11 @@ class EnableThinking(Enum):
     false = ModelDetails("False")
 
 
+class ApplyChatTemplate(Enum):
+    true = ModelDetails("True")
+    false = ModelDetails("False")
+
+
 class NumFewShots(Enum):
     shots_0 = 0
     shots_4 = 4

src/populate.py

@@ -42,6 +42,7 @@ def get_leaderboard_df(contents_repo: str, cols: list[str], benchmark_cols: list
         "llm_jp_eval_version": "llm-jp-eval version",
         "vllm_version": "vllm version",
         "enable_thinking": "Enable Thinking",
+        "apply_chat_template": "Apply Chat Template",
         "model_type": "Type",
         "model": "model_name_for_query",
     }
@@ -53,6 +54,10 @@ def get_leaderboard_df(contents_repo: str, cols: list[str], benchmark_cols: list
     if "Enable Thinking" not in df.columns:
         df["Enable Thinking"] = "False"
 
+    # Add Apply Chat Template column with default value False if it doesn't exist
+    if "Apply Chat Template" not in df.columns:
+        df["Apply Chat Template"] = "False"
+
     # Add a row ID column
     df[AutoEvalColumn.row_id.name] = range(len(df))