import gradio as gr
import pandas as pd
import plotly.graph_objects as go
import plotly.express as px
import jiwer
# Load the CSV
df = pd.read_csv("results_leaderboard.csv")
# Turkish flag banner with real flag images
banner_html = """
Turkish ASR Leaderboard
Fair and transparent comparison of Turkish ASR models
Comprehensive benchmarking on standardized test sets
"""
# About section
about_html = """
All models are evaluated on the same test sets. We calculate WER and CER values and
rank by average WER (lower = better ๐). Hover over WER values in dataset columns
to see CER values. All metrics are displayed in percentage (%).
"""
# Metrics explanations
wer_explanation = """
๐ WER โ Word Error Rate
WER measures the percentage of incorrectly recognized words.
WER = ( S + D + I ) / N
"""
cer_explanation = """
๐ CER โ Character Error Rate
CER measures the percentage of incorrectly recognized characters.
CER = ( S + D + I ) / N_char
๐ก Important for Turkish: For agglutinative languages like Turkish, CER is a more
sensitive metric than WER. A single morpheme error can affect the entire word.
"""
normalization_info = """
๐งน Normalization
Before calculation, texts are normalized as follows:
โ Convert to lowercase (ฤฐโi, Iโฤฑ - Turkish rules)
โ Remove punctuation marks
โ Normalize to single spaces
"""
# Dataset cards
dataset_cards = """
๐ฃ๏ธ CommonVoice 17.0
Crowdsourced dataset created by volunteer contributors.
Different accents and recording conditions.
๐ 9,650 recordings | Test split
๐ FLEURS Turkish
Turkish portion of Google's multilingual FLEURS dataset.
Clean speech narrated by professional speakers.
๐ 647 recordings | Test split
๐บ MDC Scripted
Mozilla Data Collection scripted speech dataset.
Turkish sentences recorded in controlled environment.
๐ 11,790 recordings | Test split
"""
# Function to format model names with medals
def format_model_with_medal(df_sorted):
df_sorted = df_sorted.copy()
df_sorted['Model'] = df_sorted.apply(
lambda row: f"๐ฅ {row['Model']}" if row.name == 0
else f"๐ฅ {row['Model']}" if row.name == 1
else f"๐ฅ {row['Model']}" if row.name == 2
else row['Model'],
axis=1
)
return df_sorted
# Function to color code model types
def color_model_type(df_formatted):
df_formatted = df_formatted.copy()
type_colors = {
'Encoder-Decoder': '๐ฆ',
'CTC': '๐ฉ',
'Transducer': '๐ง',
'Multimodal Decoder': '๐ช',
'Other': '๐จ'
}
df_formatted['Model_Type'] = df_formatted['Model_Type'].apply(
lambda x: f"{type_colors.get(x, 'โฌ')} {x}"
)
return df_formatted
# Prepare display dataframe
df_display = df.copy()
df_display = df_display.sort_values('Average_WER')
df_display = df_display.reset_index(drop=True)
df_display = format_model_with_medal(df_display)
df_display = color_model_type(df_display)
# Select and rename columns for display
display_columns = {
'Model': 'Model',
'Training_Data': 'Training Data',
'CommonVoice_WER': 'CV WER (%)',
'FLEURS_WER': 'FLEURS WER (%)',
'MDC_Scripted_WER': 'MDC WER (%)',
'Average_WER': 'Avg WER (%)',
'Params': 'Parameters',
'Model_Type': 'Type',
'License': 'License'
}
df_final = df_display[list(display_columns.keys())].rename(columns=display_columns)
# Create interactive plotly figures
def create_wer_comparison():
fig = go.Figure()
top_n = min(10, len(df_display))
# CommonVoice WER
fig.add_trace(go.Bar(
name='CommonVoice',
x=df_display['Model'][:top_n],
y=df_display['CommonVoice_WER'][:top_n],
marker_color='#E74C3C'
))
# FLEURS WER
fig.add_trace(go.Bar(
name='FLEURS',
x=df_display['Model'][:top_n],
y=df_display['FLEURS_WER'][:top_n],
marker_color='#3498DB'
))
# MDC Scripted WER
fig.add_trace(go.Bar(
name='MDC Scripted',
x=df_display['Model'][:top_n],
y=df_display['MDC_Scripted_WER'][:top_n],
marker_color='#9B59B6'
))
fig.update_layout(
title='WER Comparison - Top 10 Models',
xaxis_title='Model',
yaxis_title='WER (%)',
barmode='group',
template='plotly_white',
height=500,
font=dict(size=12)
)
return fig
def create_params_vs_performance():
import re
# Convert params to numeric (millions)
def parse_params(p):
if pd.isna(p) or p == 'TBD':
return None
# Convert to string
p_str = str(p)
# Extract all numbers with B or M
pattern = r'(\d+\.?\d*)\s*([BM])'
matches = re.findall(pattern, p_str, re.IGNORECASE)
if not matches:
return None
# Convert all to millions and take the largest
values = []
for num, unit in matches:
num_float = float(num)
if unit.upper() == 'B':
values.append(num_float * 1000)
else: # M
values.append(num_float)
return max(values) if values else None
df_plot = df_display.copy()
df_plot['Params_Numeric'] = df_plot['Params'].apply(parse_params)
df_plot = df_plot.dropna(subset=['Params_Numeric', 'Average_WER'])
fig = px.scatter(
df_plot,
x='Params_Numeric',
y='Average_WER',
text='Model',
color='Model_Type',
hover_data=['Training_Data', 'License'],
log_x=True,
title='Model Size vs Performance',
labels={
'Params_Numeric': 'Parameters (Millions)',
'Average_WER': 'Average WER (%)',
'Model_Type': 'Type'
}
)
fig.update_traces(textposition='top center', textfont_size=8)
fig.update_layout(height=600, template='plotly_white')
return fig
# WER/CER Calculator
def calculate_metrics(reference, hypothesis, normalize):
if not reference or not hypothesis:
return "N/A", "N/A"
if normalize:
# Turkish-specific normalization
import re
reference = reference.lower().replace('ฤฐ', 'i').replace('I', 'ฤฑ')
hypothesis = hypothesis.lower().replace('ฤฐ', 'i').replace('I', 'ฤฑ')
reference = re.sub(r'[^\w\s]', '', reference)
hypothesis = re.sub(r'[^\w\s]', '', hypothesis)
reference = ' '.join(reference.split())
hypothesis = ' '.join(hypothesis.split())
wer = jiwer.wer(reference, hypothesis) * 100
cer = jiwer.cer(reference, hypothesis) * 100
return f"{wer:.2f}", f"{cer:.2f}"
# Build Gradio interface
with gr.Blocks(theme=gr.themes.Soft(), css="""
.gradio-container {max-width: 1400px !important}
.markdown-text h1 {text-align: center;}
footer {visibility: hidden;}
""") as demo:
gr.HTML(banner_html)
gr.HTML(about_html)
with gr.Tabs():
with gr.Tab("๐
Leaderboard"):
gr.Markdown("### ๐ฏ Turkish ASR Models Ranking")
gr.Markdown("""
Models are ranked by **average WER** (lower = better).
๐ Top 3 models are marked with medals.
""")
leaderboard_table = gr.Dataframe(
value=df_final,
datatype=["str", "str", "number", "number", "number", "number", "str", "str", "str"],
interactive=False,
wrap=True
)
with gr.Tab("๐ Metrics & Calculator"):
gr.Markdown("## Evaluation Metrics Explained")
gr.HTML(wer_explanation)
gr.HTML(cer_explanation)
gr.HTML(normalization_info)
gr.Markdown("---")
gr.Markdown("### ๐ค Sandbox: Calculate WER/CER on Your Own Texts")
gr.Markdown("Calculate WER and CER metrics on your own text samples!")
with gr.Row():
with gr.Column():
ref_text = gr.Textbox(
label="๐ Reference Text",
placeholder="Enter correct text here...",
lines=5
)
with gr.Column():
hyp_text = gr.Textbox(
label="๐ค Recognized Text",
placeholder="Enter model output here...",
lines=5
)
normalize_check = gr.Checkbox(
label="๐งน Normalize (lowercase, remove punctuation)",
value=True
)
calc_btn = gr.Button("๐ข Calculate", variant="primary", size="lg")
with gr.Row():
wer_output = gr.Textbox(label="WER, %", interactive=False)
cer_output = gr.Textbox(label="CER, %", interactive=False)
calc_btn.click(
fn=calculate_metrics,
inputs=[ref_text, hyp_text, normalize_check],
outputs=[wer_output, cer_output]
)
gr.Markdown("""
**๐ก Tip:** Normalization ignores case and punctuation differences.
This is standard practice in real ASR evaluation.
""")
with gr.Tab("๐ Datasets"):
gr.Markdown("## ๐ฆ Test Datasets")
gr.Markdown("""
All results are calculated on the same test sets.
Different datasets show model performance across various speech types.
""")
gr.HTML(dataset_cards)
with gr.Tab("๐ Visualizations"):
gr.Markdown("### Performance Comparisons")
with gr.Row():
gr.Plot(value=create_wer_comparison())
with gr.Row():
gr.Plot(value=create_params_vs_performance())
with gr.Tab("โน๏ธ About"):
gr.Markdown("""
### ๐น๐ท About Turkish ASR Leaderboard
This platform is designed to provide **fair, transparent, and reproducible**
comparison of Turkish speech recognition models.
**๐ฌ Methodology:**
- All models tested on **same hardware** (NVIDIA L40S GPU)
- **Standardized test sets** are used
- Metrics calculated using **jiwer** library
- **No cherry-picking**: All results shared transparently
---
### ๐ Submit Your Model
Have you developed a Turkish ASR model? Let's add it to the leaderboard!
**Required Information:**
1. ๐ **WER/CER Scores**: For each test set (CommonVoice, FLEURS, MDC Scripted)
2. ๐ข **Parameters**: Model size
3. ๐ **Training Data**: Which dataset was used for training?
4. ๐ **License**: License type
**Submission Channels:**
- ๐ค **Hugging Face**: Comment in this space's discussion section
- ๐ง **Contact**: [@y0mur](https://huggingface.co/y0mur)
---
**๐ Acknowledgments:**
- **Hugging Face** - Hosting and infrastructure
- **OpenAI** - Whisper models
- **Meta AI** - Wav2Vec2 and MMS models
- **Mistral AI** - Voxtral base model
- **Turkish NLP Community** - Datasets and support
**๐ License:** MIT
**๐ค Maintainer:** [y0mur](https://huggingface.co/y0mur)
---
Last Updated: December 2025
""")
gr.Markdown("""
""")
if __name__ == "__main__":
demo.launch()