import os import traceback import gradio as gr import pandas as pd import plotly.express as px import plotly.graph_objects as go from apscheduler.schedulers.background import BackgroundScheduler from huggingface_hub import snapshot_download from src.about import ( BOTTOM_LOGO, CITATION_BUTTON_LABEL, CITATION_BUTTON_LABEL_JA, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, EVALUATION_QUEUE_TEXT_JA, INTRODUCTION_TEXT, INTRODUCTION_TEXT_JA, LLM_BENCHMARKS_TEXT, LLM_BENCHMARKS_TEXT_JA, TITLE, TaskType, ) from src.display.utils import ( BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, NUMERIC_INTERVALS, TYPES, AddSpecialTokens, ApplyChatTemplate, AutoEvalColumn, EnableThinking, ModelType, Precision, fields, ) from src.envs import API, CONTENTS_REPO, EVAL_REQUESTS_PATH, QUEUE_REPO, REPO_ID from src.i18n import ( CITATION_ACCORDION_LABEL, CITATION_ACCORDION_LABEL_JA, SELECT_ALL_BUTTON_LABEL, SELECT_ALL_BUTTON_LABEL_JA, SELECT_AVG_ONLY_BUTTON_LABEL, SELECT_AVG_ONLY_BUTTON_LABEL_JA, SELECT_NONE_BUTTON_LABEL, SELECT_NONE_BUTTON_LABEL_JA, ) from src.populate import get_evaluation_queue_df, get_leaderboard_df from src.submission.submit import cancel_eval, confirm_eval, preview_eval def restart_space() -> None: API.restart_space(repo_id=REPO_ID) # Queue sync is best-effort β€” the leaderboard can still serve without it. try: snapshot_download( repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=API.token, ) except Exception as e: print( f"[startup] Failed to sync eval queue from {QUEUE_REPO}: " f"{type(e).__name__}: {e}", flush=True, ) traceback.print_exc() EVAL_REQUESTS_PATH.mkdir(parents=True, exist_ok=True) # Get dataframes try: ( FINISHED_EVAL_QUEUE_DF, RUNNING_EVAL_QUEUE_DF, PENDING_EVAL_QUEUE_DF, FAILED_EVAL_QUEUE_DF, ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) except Exception as e: print( f"[startup] Failed to build eval queue df from {EVAL_REQUESTS_PATH}: " f"{type(e).__name__}: {e}", flush=True, ) traceback.print_exc() _empty_queue_df = pd.DataFrame(columns=EVAL_COLS) FINISHED_EVAL_QUEUE_DF = _empty_queue_df RUNNING_EVAL_QUEUE_DF = _empty_queue_df PENDING_EVAL_QUEUE_DF = _empty_queue_df FAILED_EVAL_QUEUE_DF = _empty_queue_df try: ORIGINAL_DF = get_leaderboard_df(CONTENTS_REPO, COLS, BENCHMARK_COLS) except Exception as e: print( f"[startup] Failed to load leaderboard from {CONTENTS_REPO}: " f"{type(e).__name__}: {e}", flush=True, ) traceback.print_exc() ORIGINAL_DF = pd.DataFrame() # Searching and filtering def filter_models( df: pd.DataFrame, type_query: list[str], size_query: list[str], precision_query: list[str], add_special_tokens_query: list[str], enable_thinking_query: list[str], apply_chat_template_query: list[str], ) -> pd.DataFrame: # Filter by model type type_emoji = [t.split()[0] for t in type_query] df = df[df["T"].isin(type_emoji)] # Filter by precision df = df[df["Precision"].isin(precision_query)] # Filter by model size # Note: When `df` is empty, `size_mask` is empty, and the shape of `df[size_mask]` becomes (0, 0), # so we need to check the length of `df` before applying the filter. if len(df) > 0: size_mask = df["#Params (B)"].apply( lambda x: any(x in NUMERIC_INTERVALS[s] for s in size_query if s != "Unknown") ) if "Unknown" in size_query: size_mask |= df["#Params (B)"].isna() | (df["#Params (B)"] == 0) df = df[size_mask] # Filter by special tokens setting df = df[df["Add Special Tokens"].isin(add_special_tokens_query)] # Filter by enable_thinking df = df[df["Enable Thinking"].isin(enable_thinking_query)] # Filter by apply_chat_template df = df[df["Apply Chat Template"].isin(apply_chat_template_query)] return df def search_model_by_name(df: pd.DataFrame, model_name: str) -> pd.DataFrame: return df[df[AutoEvalColumn.dummy.name].str.contains(model_name, case=False)] def search_models_by_multiple_names(df: pd.DataFrame, search_text: str) -> pd.DataFrame: if not search_text: return df model_names = [name.strip() for name in search_text.split(";")] dfs = [search_model_by_name(df, name) for name in model_names if name] return pd.concat(dfs).drop_duplicates(subset=AutoEvalColumn.row_id.name) def select_columns(df: pd.DataFrame, columns: list[str]) -> pd.DataFrame: always_here_cols = [ AutoEvalColumn.model_type_symbol.name, # 'T' AutoEvalColumn.model.name, # 'Model' ] # Remove 'always_here_cols' from 'columns' to avoid duplicates columns = [c for c in columns if c not in always_here_cols] new_columns = ( always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.row_id.name] ) # Maintain order while removing duplicates seen = set() unique_columns = [] for c in new_columns: if c not in seen: unique_columns.append(c) seen.add(c) # Create DataFrame with filtered columns filtered_df = df[unique_columns] return filtered_df def update_table( type_query: list[str], precision_query: list[str], size_query: list[str], add_special_tokens_query: list[str], enable_thinking_query: list[str], apply_chat_template_query: list[str], query: str, *columns, ) -> pd.DataFrame: columns = [item for column in columns for item in column] df = filter_models( ORIGINAL_DF, type_query, size_query, precision_query, add_special_tokens_query, enable_thinking_query, apply_chat_template_query, ) df = search_models_by_multiple_names(df, query) df = select_columns(df, columns) return df # Prepare the dataframes INITIAL_COLUMNS = ["T"] + [ c.name for c in fields(AutoEvalColumn) if (c.never_hidden or c.displayed_by_default) and c.name != "T" ] leaderboard_df = ORIGINAL_DF.copy() if len(leaderboard_df) > 0: leaderboard_df = filter_models( leaderboard_df, [t.to_str(" : ") for t in ModelType], list(NUMERIC_INTERVALS.keys()), [i.value.name for i in Precision], [i.value.name for i in AddSpecialTokens], [i.value.name for i in EnableThinking], [i.value.name for i in ApplyChatTemplate], ) leaderboard_df = select_columns(leaderboard_df, INITIAL_COLUMNS) else: leaderboard_df = pd.DataFrame(columns=INITIAL_COLUMNS) # Leaderboard demo def toggle_all_categories(action: str) -> list[gr.CheckboxGroup]: """Function to control all category checkboxes at once""" results = [] for task_type in TaskType: if task_type == TaskType.NotTask: # Maintain existing selection for Model details results.append(gr.CheckboxGroup()) elif action == "all": # Select all results.append( gr.CheckboxGroup( value=[ c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and not c.dummy and c.task_type == task_type ] ) ) elif action == "none": # Deselect all results.append(gr.CheckboxGroup(value=[])) elif action == "avg_only": # Select only AVG metrics results.append( gr.CheckboxGroup( value=[ c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and c.task_type == task_type and ((task_type == TaskType.AVG) or (task_type != TaskType.AVG and c.average)) ] ) ) return results TASK_AVG_NAME_MAP = { c.name: c.task_type.name for c in fields(AutoEvalColumn) if c.average and c.task_type != TaskType.AVG } AVG_COLUMNS = ["AVG"] + list(TASK_AVG_NAME_MAP.keys()) def plot_size_vs_score(df_filtered: pd.DataFrame) -> go.Figure: if len(ORIGINAL_DF) == 0 or AutoEvalColumn.row_id.name not in ORIGINAL_DF.columns: return go.Figure() df = ORIGINAL_DF[ORIGINAL_DF[AutoEvalColumn.row_id.name].isin(df_filtered[AutoEvalColumn.row_id.name])] df = df[df["#Params (B)"] > 0] available_avg = [c for c in AVG_COLUMNS if c in df.columns] df = df[["model_name_for_query", "#Params (B)"] + available_avg] df = df.rename(columns={"model_name_for_query": "Model"}) df["model_name_without_org_name"] = df["Model"].str.split("/").str[-1] df = pd.melt( df, id_vars=["Model", "model_name_without_org_name", "#Params (B)"], value_vars=available_avg, var_name="Category", value_name="Score", ) max_model_size = df["#Params (B)"].max() fig = px.scatter( df, x="#Params (B)", y="Score", text="model_name_without_org_name", color="Category", hover_data=["Model", "Category"], ) fig.update_traces( hovertemplate="%{customdata[0]}
#Params: %{x:.2f}B
%{customdata[1]}: %{y:.4f}", textposition="top right", mode="markers+text", ) for trace in fig.data: if trace.name != "AVG": trace.visible = "legendonly" fig.update_layout(xaxis_range=[0, max_model_size * 1.2], yaxis_range=[0, 1]) fig.update_layout( updatemenus=[ { "type": "buttons", "direction": "left", "showactive": True, "buttons": [ {"label": "Hide Labels", "method": "restyle", "args": ["mode", "markers"]}, {"label": "Show Labels", "method": "restyle", "args": ["mode", "markers+text"]}, ], "x": 0.5, "y": -0.2, "xanchor": "center", "yanchor": "top", } ] ) return fig def plot_average_scores(df_filtered: pd.DataFrame) -> go.Figure: if len(ORIGINAL_DF) == 0 or AutoEvalColumn.row_id.name not in ORIGINAL_DF.columns: return go.Figure() df = ORIGINAL_DF[ORIGINAL_DF[AutoEvalColumn.row_id.name].isin(df_filtered[AutoEvalColumn.row_id.name])] available_avg_keys = [k for k in TASK_AVG_NAME_MAP if k in df.columns] df = df[["model_name_for_query"] + available_avg_keys] df = df.rename(columns={"model_name_for_query": "Model"}) df = df.rename(columns={k: TASK_AVG_NAME_MAP[k] for k in available_avg_keys}) df = df.set_index("Model") fig = go.Figure() for i, (name, row) in enumerate(df.iterrows()): visible = True if i < 2 else "legendonly" # Display only the first 2 models fig.add_trace( go.Scatterpolar( r=row.values, theta=row.index, fill="toself", name=name, hovertemplate="%{theta}: %{r}", visible=visible, ) ) fig.update_layout( polar={ "radialaxis": {"range": [0, 1]}, }, showlegend=True, ) return fig shown_columns_dict: dict[str, gr.CheckboxGroup] = {} checkboxes: list[gr.CheckboxGroup] = [] with gr.Blocks() as demo_leaderboard: with gr.Row(): search_bar = gr.Textbox( placeholder=" πŸ” Search for your model (separate multiple queries with `;`) and press ENTER...", show_label=False, elem_id="search-bar", ) with gr.Accordion("Column Filter", open=True): with gr.Row(): with gr.Row(): select_all_button = gr.Button(SELECT_ALL_BUTTON_LABEL_JA, size="sm") select_none_button = gr.Button(SELECT_NONE_BUTTON_LABEL_JA, size="sm") select_avg_only_button = gr.Button(SELECT_AVG_ONLY_BUTTON_LABEL_JA, size="sm") for task_type in TaskType: label = "Model details" if task_type == TaskType.NotTask else task_type.value with gr.Accordion(label, open=True, elem_classes="accordion"): with gr.Row(height=110): shown_column = gr.CheckboxGroup( show_label=False, choices=[ c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and not c.dummy and c.task_type == task_type ], value=[ c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and c.task_type == task_type ], elem_id="column-select", container=False, ) shown_columns_dict[task_type.name] = shown_column checkboxes.append(shown_column) with gr.Accordion("Model Filter", open=True): with gr.Row(): filter_columns_type = gr.CheckboxGroup( label="Model types", choices=[t.to_str() for t in ModelType], value=[t.to_str() for t in ModelType], elem_id="filter-columns-type", ) filter_columns_precision = gr.CheckboxGroup( label="Precision", choices=[i.value.name for i in Precision], value=[i.value.name for i in Precision], elem_id="filter-columns-precision", ) filter_columns_size = gr.CheckboxGroup( label="Model sizes (in billions of parameters)", choices=list(NUMERIC_INTERVALS.keys()), value=list(NUMERIC_INTERVALS.keys()), elem_id="filter-columns-size", ) filter_columns_add_special_tokens = gr.CheckboxGroup( label="Add Special Tokens", choices=[i.value.name for i in AddSpecialTokens], value=[i.value.name for i in AddSpecialTokens], elem_id="filter-columns-add-special-tokens", ) filter_columns_enable_thinking = gr.CheckboxGroup( label="Enable Thinking", choices=[i.value.name for i in EnableThinking], value=[i.value.name for i in EnableThinking], elem_id="filter-columns-enable-thinking", ) filter_columns_apply_chat_template = gr.CheckboxGroup( label="Apply Chat Template", choices=[i.value.name for i in ApplyChatTemplate], value=[i.value.name for i in ApplyChatTemplate], elem_id="filter-columns-apply-chat-template", ) leaderboard_table = gr.Dataframe( value=leaderboard_df, headers=INITIAL_COLUMNS, datatype=TYPES, elem_id="leaderboard-table", interactive=False, visible=True, ) graph_size_vs_score = gr.Plot(label="Size vs. Score", value=plot_size_vs_score(leaderboard_df)) graph_average_scores = gr.Plot( label="Performance across Task Categories", value=plot_average_scores(leaderboard_df) ) select_all_button.click( fn=lambda: toggle_all_categories("all"), outputs=checkboxes, api_name=False, queue=False, ) select_none_button.click( fn=lambda: toggle_all_categories("none"), outputs=checkboxes, api_name=False, queue=False, ) select_avg_only_button.click( fn=lambda: toggle_all_categories("avg_only"), outputs=checkboxes, api_name=False, queue=False, ) gr.on( triggers=[ filter_columns_type.change, filter_columns_precision.change, filter_columns_size.change, filter_columns_add_special_tokens.change, filter_columns_enable_thinking.change, filter_columns_apply_chat_template.change, search_bar.submit, ] + [shown_columns.change for shown_columns in shown_columns_dict.values()], fn=update_table, inputs=[ filter_columns_type, filter_columns_precision, filter_columns_size, filter_columns_add_special_tokens, filter_columns_enable_thinking, filter_columns_apply_chat_template, search_bar, ] + list(shown_columns_dict.values()), outputs=leaderboard_table, ) leaderboard_table.change( fn=plot_size_vs_score, inputs=leaderboard_table, outputs=graph_size_vs_score, api_name=False, queue=False, ) leaderboard_table.change( fn=plot_average_scores, inputs=leaderboard_table, outputs=graph_average_scores, api_name=False, queue=False, ) # Submission demo def display_user_info(profile: gr.OAuthProfile | None) -> str: """Display user information if logged in""" if profile is None: return "Please log in to submit a model" return f"Logged in as: **{profile.name}** (@{profile.username})" with gr.Blocks() as demo_submission: with gr.Column(): with gr.Row(): evaluation_queue_text = gr.Markdown(EVALUATION_QUEUE_TEXT_JA, elem_classes="markdown-text") with gr.Column(): with gr.Accordion( f"βœ… Finished Evaluations ({len(FINISHED_EVAL_QUEUE_DF)})", open=False, ): with gr.Row(): finished_eval_table = gr.Dataframe( value=FINISHED_EVAL_QUEUE_DF, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5, ) with gr.Accordion( f"πŸ”„ Running Evaluation Queue ({len(RUNNING_EVAL_QUEUE_DF)})", open=True, ): with gr.Row(): running_eval_table = gr.Dataframe( value=RUNNING_EVAL_QUEUE_DF, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5, ) with gr.Accordion( f"⏳ Pending Evaluation Queue ({len(PENDING_EVAL_QUEUE_DF)})", open=False, ): with gr.Row(): pending_eval_table = gr.Dataframe( value=PENDING_EVAL_QUEUE_DF, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5, ) with gr.Accordion( f"❎ Failed Evaluation Queue ({len(FAILED_EVAL_QUEUE_DF)})", open=False, ): with gr.Row(): failed_eval_table = gr.Dataframe( value=FAILED_EVAL_QUEUE_DF, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5, ) with gr.Row(): with gr.Column(scale=3): gr.Markdown("# βœ‰οΈβœ¨ Submit your model here!", elem_classes="markdown-text") with gr.Row(): user_info_markdown = gr.Markdown("Please log in to submit a model", elem_classes="markdown-text") with gr.Row(): gr.LoginButton(size="lg") with gr.Row(): with gr.Column(): model_name_textbox = gr.Textbox(label="Model name") revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main") model_type = gr.Dropdown( label="Model type", choices=[t.to_str(" : ") for t in ModelType], multiselect=False, value=None, ) with gr.Column(): precision = gr.Dropdown( label="Precision", choices=[i.value.name for i in Precision] + ["auto"], multiselect=False, value="auto", ) add_special_tokens = gr.Dropdown( label="AddSpecialTokens", choices=[i.value.name for i in AddSpecialTokens], multiselect=False, value="False", ) with gr.Row(): with gr.Column(): apply_chat_template = gr.Dropdown( label="Apply Chat Template", choices=["False", "True"], multiselect=False, value="True", info="Whether to apply chat template to the model", ) enable_thinking = gr.Dropdown( label="Enable Thinking (Reasoning mode)", choices=["False", "True"], multiselect=False, value="False", info="Enable thinking mode (only for compatible models like Qwen3). Max reasoning length: 1024 tokens", ) with gr.Column(): reasoning_parser = gr.Dropdown( label="Reasoning Parser", choices=[ "", "deepseek_r1", "deepseek_v3", "ernie45", "gemma4", "glm45", "openai_gptoss", "granite", "hunyuan_a13b", "kimi_k2", "llmjp4", "minimax_m2", "minimax_m2_append_think", "mistral", "olmo3", "qwen3", "seed_oss", "step3", ], multiselect=False, value="", info="Reasoning parser type (only effective when Enable Thinking=True)", ) submit_button = gr.Button("Submit Eval") submission_result = gr.Markdown() eval_entry_state = gr.State(value=None) with gr.Column(visible=False, elem_classes="confirm-modal-overlay") as modal_overlay: with gr.Column(elem_classes="confirm-modal-box"): modal_content = gr.Markdown() with gr.Row(): confirm_button = gr.Button("Confirm", variant="primary", scale=1) cancel_button = gr.Button("Cancel", variant="secondary", scale=1) submit_inputs = [ model_name_textbox, revision_name_textbox, precision, model_type, add_special_tokens, apply_chat_template, enable_thinking, reasoning_parser, ] submit_button.click( fn=preview_eval, inputs=submit_inputs, outputs=[submission_result, modal_content, modal_overlay, eval_entry_state], api_name="preview_eval", ) confirm_button.click( fn=confirm_eval, inputs=[eval_entry_state], outputs=[submission_result, modal_overlay, eval_entry_state], api_name="confirm_eval", ) cancel_button.click( fn=cancel_eval, inputs=[], outputs=[submission_result, modal_overlay, eval_entry_state], ) # Load user info when the page loads demo_submission.load(fn=display_user_info, outputs=user_info_markdown) # Main demo def set_default_language(request: gr.Request) -> gr.Radio: if request.headers["Accept-Language"].split(",")[0].lower().startswith("ja"): return gr.Radio(value="πŸ‡―πŸ‡΅ JA") else: return gr.Radio(value="πŸ‡ΊπŸ‡Έ EN") def update_language( language: str, ) -> tuple[ gr.Markdown, # introduction_text gr.Markdown, # llm_benchmarks_text gr.Markdown, # evaluation_queue_text gr.Textbox, # citation_button gr.Button, # select_all_button gr.Button, # select_none_button gr.Button, # select_avg_only_button gr.Accordion, # citation_accordion ]: if language == "πŸ‡―πŸ‡΅ JA": return ( gr.Markdown(value=INTRODUCTION_TEXT_JA), gr.Markdown(value=LLM_BENCHMARKS_TEXT_JA), gr.Markdown(value=EVALUATION_QUEUE_TEXT_JA), gr.Textbox(label=CITATION_BUTTON_LABEL_JA), gr.Button(value=SELECT_ALL_BUTTON_LABEL_JA), gr.Button(value=SELECT_NONE_BUTTON_LABEL_JA), gr.Button(value=SELECT_AVG_ONLY_BUTTON_LABEL_JA), gr.Accordion(label=CITATION_ACCORDION_LABEL_JA), ) else: return ( gr.Markdown(value=INTRODUCTION_TEXT), gr.Markdown(value=LLM_BENCHMARKS_TEXT), gr.Markdown(value=EVALUATION_QUEUE_TEXT), gr.Textbox(label=CITATION_BUTTON_LABEL), gr.Button(value=SELECT_ALL_BUTTON_LABEL), gr.Button(value=SELECT_NONE_BUTTON_LABEL), gr.Button(value=SELECT_AVG_ONLY_BUTTON_LABEL), gr.Accordion(label=CITATION_ACCORDION_LABEL), ) with gr.Blocks(css_paths="style.css", theme=gr.themes.Glass()) as demo: gr.HTML(TITLE) introduction_text = gr.Markdown(INTRODUCTION_TEXT_JA, elem_classes="markdown-text") with gr.Tabs() as tabs: with gr.Tab("πŸ… LLM Benchmark", elem_id="llm-benchmark-tab-table"): demo_leaderboard.render() with gr.Tab("πŸ“ About", elem_id="llm-benchmark-tab-about"): llm_benchmarks_text = gr.Markdown(LLM_BENCHMARKS_TEXT_JA, elem_classes="markdown-text") with gr.Tab("πŸš€ Submit here! ", elem_id="llm-benchmark-tab-submit"): demo_submission.render() with gr.Row(): with gr.Accordion(CITATION_ACCORDION_LABEL_JA, open=False) as citation_accordion: citation_button = gr.Textbox( label=CITATION_BUTTON_LABEL_JA, value=CITATION_BUTTON_TEXT, lines=20, elem_id="citation-button", show_copy_button=True, ) gr.HTML(BOTTOM_LOGO) language = gr.Radio( choices=["πŸ‡―πŸ‡΅ JA", "πŸ‡ΊπŸ‡Έ EN"], value="πŸ‡―πŸ‡΅ JA", elem_classes="language-selector", show_label=False, container=False, ) demo.load(fn=set_default_language, outputs=language) language.change( fn=update_language, inputs=language, outputs=[ introduction_text, llm_benchmarks_text, evaluation_queue_text, citation_button, select_all_button, select_none_button, select_avg_only_button, citation_accordion, ], api_name=False, ) if __name__ == "__main__": if os.getenv("SPACE_ID"): scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=1800) scheduler.start() demo.queue(default_concurrency_limit=40).launch(ssr_mode=False)