Spaces:
Runtime error
Runtime error
temp migration
Browse files- app.py +90 -63
- graphs/leaderboard.py +150 -113
app.py
CHANGED
|
@@ -59,8 +59,8 @@ con.execute("SET enable_object_cache = false;")
|
|
| 59 |
|
| 60 |
# Load parquet files from Hugging Face using DuckDB
|
| 61 |
HF_DATASET_ID = "mmpr/open_model_evolution_data"
|
| 62 |
-
hf_parquet_url_1 = "https://huggingface.co/datasets/
|
| 63 |
-
hf_parquet_url_2 = "https://huggingface.co/datasets/
|
| 64 |
|
| 65 |
print(f"Attempting to connect to dataset from Hugging Face Hub: {HF_DATASET_ID}")
|
| 66 |
try:
|
|
@@ -745,17 +745,26 @@ def _get_filtered_top_n_from_duckdb(
|
|
| 745 |
slider_value, group_col, top_n, view="all_downloads"
|
| 746 |
):
|
| 747 |
"""
|
| 748 |
-
Query DuckDB
|
| 749 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 750 |
"""
|
| 751 |
-
|
| 752 |
-
|
| 753 |
if slider_value and len(slider_value) == 2:
|
| 754 |
start = pd.to_datetime(slider_value[0], unit="s")
|
| 755 |
end = pd.to_datetime(slider_value[1], unit="s")
|
| 756 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 757 |
|
| 758 |
-
# If grouping by country,
|
| 759 |
if group_col == "org_country_single":
|
| 760 |
group_expr = """CASE
|
| 761 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
|
@@ -765,12 +774,11 @@ def _get_filtered_top_n_from_duckdb(
|
|
| 765 |
else:
|
| 766 |
group_expr = group_col
|
| 767 |
|
| 768 |
-
#
|
| 769 |
-
# When grouping by derived_author, we need to find the country where derived_author = author
|
| 770 |
if group_col == "derived_author":
|
| 771 |
query = f"""
|
| 772 |
WITH base_data AS (
|
| 773 |
-
SELECT
|
| 774 |
{group_expr} AS group_key,
|
| 775 |
CASE
|
| 776 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
|
@@ -781,93 +789,112 @@ def _get_filtered_top_n_from_duckdb(
|
|
| 781 |
derived_author,
|
| 782 |
merged_country_groups_single,
|
| 783 |
merged_modality,
|
| 784 |
-
|
| 785 |
-
|
|
|
|
| 786 |
FROM {view}
|
| 787 |
-
{time_clause}
|
| 788 |
),
|
| 789 |
-
|
| 790 |
-
-- Create a lookup table for derived_author -> country
|
| 791 |
author_country_lookup AS (
|
| 792 |
SELECT DISTINCT
|
| 793 |
author,
|
| 794 |
-
FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY
|
| 795 |
FROM base_data
|
| 796 |
WHERE author IS NOT NULL
|
| 797 |
),
|
| 798 |
|
| 799 |
-
|
| 800 |
-
SELECT
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 801 |
FROM base_data
|
|
|
|
| 802 |
),
|
| 803 |
|
| 804 |
-
|
| 805 |
-
SELECT
|
| 806 |
-
b.group_key AS name,
|
| 807 |
-
SUM(b.downloads) AS total_downloads,
|
| 808 |
-
ROUND(SUM(b.downloads) * 100.0 / t.total_downloads_all, 2) AS percent_of_total,
|
| 809 |
-
COALESCE(acl.author_country, ANY_VALUE(b.org_country_single)) AS org_country_single,
|
| 810 |
-
ANY_VALUE(b.author) AS author,
|
| 811 |
-
ANY_VALUE(b.derived_author) AS derived_author,
|
| 812 |
-
ANY_VALUE(b.merged_country_groups_single) AS merged_country_groups_single,
|
| 813 |
-
ANY_VALUE(b.merged_modality) AS merged_modality,
|
| 814 |
-
ANY_VALUE(b.model) AS model
|
| 815 |
-
FROM base_data b
|
| 816 |
-
CROSS JOIN total_downloads_cte t
|
| 817 |
-
LEFT JOIN author_country_lookup acl ON b.group_key = acl.author
|
| 818 |
-
GROUP BY b.group_key, acl.author_country, t.total_downloads_all
|
| 819 |
)
|
| 820 |
|
| 821 |
-
SELECT
|
| 822 |
-
|
| 823 |
-
|
| 824 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 825 |
"""
|
| 826 |
else:
|
| 827 |
query = f"""
|
| 828 |
WITH base_data AS (
|
| 829 |
-
SELECT
|
| 830 |
{group_expr} AS group_key,
|
| 831 |
CASE
|
| 832 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
| 833 |
-
WHEN org_country_single IN ('International', 'Online') THEN 'International/Online'
|
| 834 |
ELSE org_country_single
|
| 835 |
END AS org_country_single,
|
| 836 |
author,
|
| 837 |
derived_author,
|
| 838 |
merged_country_groups_single,
|
| 839 |
merged_modality,
|
| 840 |
-
|
| 841 |
-
|
|
|
|
| 842 |
FROM {view}
|
| 843 |
-
{time_clause}
|
| 844 |
),
|
| 845 |
|
| 846 |
-
|
| 847 |
-
SELECT
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 848 |
FROM base_data
|
|
|
|
| 849 |
),
|
| 850 |
|
| 851 |
-
|
| 852 |
-
SELECT
|
| 853 |
-
b.group_key AS name,
|
| 854 |
-
SUM(b.downloads) AS total_downloads,
|
| 855 |
-
ROUND(SUM(b.downloads) * 100.0 / t.total_downloads_all, 2) AS percent_of_total,
|
| 856 |
-
ANY_VALUE(b.org_country_single) AS org_country_single,
|
| 857 |
-
ANY_VALUE(b.author) AS author,
|
| 858 |
-
ANY_VALUE(b.derived_author) AS derived_author,
|
| 859 |
-
ANY_VALUE(b.merged_country_groups_single) AS merged_country_groups_single,
|
| 860 |
-
ANY_VALUE(b.merged_modality) AS merged_modality,
|
| 861 |
-
ANY_VALUE(b.model) AS model
|
| 862 |
-
FROM base_data b
|
| 863 |
-
CROSS JOIN total_downloads_cte t
|
| 864 |
-
GROUP BY b.group_key, t.total_downloads_all
|
| 865 |
)
|
| 866 |
|
| 867 |
-
SELECT
|
| 868 |
-
|
| 869 |
-
|
| 870 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 871 |
"""
|
| 872 |
|
| 873 |
return con.execute(query).fetchdf()
|
|
|
|
| 59 |
|
| 60 |
# Load parquet files from Hugging Face using DuckDB
|
| 61 |
HF_DATASET_ID = "mmpr/open_model_evolution_data"
|
| 62 |
+
hf_parquet_url_1 = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/all_downloads_with_annotations-1.parquet"
|
| 63 |
+
hf_parquet_url_2 = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/one_year_rolling-2.parquet"
|
| 64 |
|
| 65 |
print(f"Attempting to connect to dataset from Hugging Face Hub: {HF_DATASET_ID}")
|
| 66 |
try:
|
|
|
|
| 745 |
slider_value, group_col, top_n, view="all_downloads"
|
| 746 |
):
|
| 747 |
"""
|
| 748 |
+
Query DuckDB to get model-level rows with per-model total_downloads (delta or full)
|
| 749 |
+
Returns a DataFrame with columns including:
|
| 750 |
+
- group_key (the grouping column)
|
| 751 |
+
- org_country_single, author, derived_author, merged_country_groups_single, merged_modality, model
|
| 752 |
+
- total_downloads (per-model downloads in requested window)
|
| 753 |
+
- percent_of_total (percent of total across all returned model deltas)
|
| 754 |
"""
|
| 755 |
+
|
| 756 |
+
# Compute date window (if slider_value provided, use it; otherwise cover full range)
|
| 757 |
if slider_value and len(slider_value) == 2:
|
| 758 |
start = pd.to_datetime(slider_value[0], unit="s")
|
| 759 |
end = pd.to_datetime(slider_value[1], unit="s")
|
| 760 |
+
else:
|
| 761 |
+
start = pd.to_datetime("1970-01-01")
|
| 762 |
+
end = end_dt # defined near top of file when parquet was loaded
|
| 763 |
+
|
| 764 |
+
start_str = str(start)
|
| 765 |
+
end_str = str(end)
|
| 766 |
|
| 767 |
+
# If grouping by country, transform some country values
|
| 768 |
if group_col == "org_country_single":
|
| 769 |
group_expr = """CASE
|
| 770 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
|
|
|
| 774 |
else:
|
| 775 |
group_expr = group_col
|
| 776 |
|
| 777 |
+
# Derived-author requires author->country lookup; build separate SQL for that case
|
|
|
|
| 778 |
if group_col == "derived_author":
|
| 779 |
query = f"""
|
| 780 |
WITH base_data AS (
|
| 781 |
+
SELECT
|
| 782 |
{group_expr} AS group_key,
|
| 783 |
CASE
|
| 784 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
|
|
|
| 789 |
derived_author,
|
| 790 |
merged_country_groups_single,
|
| 791 |
merged_modality,
|
| 792 |
+
model,
|
| 793 |
+
time,
|
| 794 |
+
downloadsAllTime
|
| 795 |
FROM {view}
|
|
|
|
| 796 |
),
|
| 797 |
+
|
|
|
|
| 798 |
author_country_lookup AS (
|
| 799 |
SELECT DISTINCT
|
| 800 |
author,
|
| 801 |
+
FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloadsAllTime DESC) AS author_country
|
| 802 |
FROM base_data
|
| 803 |
WHERE author IS NOT NULL
|
| 804 |
),
|
| 805 |
|
| 806 |
+
model_metrics AS (
|
| 807 |
+
SELECT
|
| 808 |
+
model,
|
| 809 |
+
group_key,
|
| 810 |
+
ANY_VALUE(org_country_single) AS org_country_single,
|
| 811 |
+
ANY_VALUE(author) AS author,
|
| 812 |
+
ANY_VALUE(derived_author) AS derived_author,
|
| 813 |
+
ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
|
| 814 |
+
ANY_VALUE(merged_modality) AS merged_modality,
|
| 815 |
+
COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
|
| 816 |
+
- COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
|
| 817 |
+
AS total_downloads
|
| 818 |
FROM base_data
|
| 819 |
+
GROUP BY model, group_key
|
| 820 |
),
|
| 821 |
|
| 822 |
+
total_downloads_cte AS (
|
| 823 |
+
SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 824 |
)
|
| 825 |
|
| 826 |
+
SELECT
|
| 827 |
+
mm.model,
|
| 828 |
+
mm.group_key,
|
| 829 |
+
COALESCE(acl.author_country, mm.org_country_single) AS org_country_single,
|
| 830 |
+
mm.author,
|
| 831 |
+
mm.derived_author,
|
| 832 |
+
mm.merged_country_groups_single,
|
| 833 |
+
mm.merged_modality,
|
| 834 |
+
mm.total_downloads,
|
| 835 |
+
CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
|
| 836 |
+
FROM model_metrics mm
|
| 837 |
+
LEFT JOIN author_country_lookup acl ON mm.group_key = acl.author
|
| 838 |
+
CROSS JOIN total_downloads_cte td
|
| 839 |
+
WHERE mm.total_downloads > 0
|
| 840 |
+
ORDER BY mm.total_downloads DESC
|
| 841 |
+
LIMIT {top_n * 10};
|
| 842 |
"""
|
| 843 |
else:
|
| 844 |
query = f"""
|
| 845 |
WITH base_data AS (
|
| 846 |
+
SELECT
|
| 847 |
{group_expr} AS group_key,
|
| 848 |
CASE
|
| 849 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
| 850 |
+
WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
|
| 851 |
ELSE org_country_single
|
| 852 |
END AS org_country_single,
|
| 853 |
author,
|
| 854 |
derived_author,
|
| 855 |
merged_country_groups_single,
|
| 856 |
merged_modality,
|
| 857 |
+
model,
|
| 858 |
+
time,
|
| 859 |
+
downloadsAllTime
|
| 860 |
FROM {view}
|
|
|
|
| 861 |
),
|
| 862 |
|
| 863 |
+
model_metrics AS (
|
| 864 |
+
SELECT
|
| 865 |
+
model,
|
| 866 |
+
group_key,
|
| 867 |
+
ANY_VALUE(org_country_single) AS org_country_single,
|
| 868 |
+
ANY_VALUE(author) AS author,
|
| 869 |
+
ANY_VALUE(derived_author) AS derived_author,
|
| 870 |
+
ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
|
| 871 |
+
ANY_VALUE(merged_modality) AS merged_modality,
|
| 872 |
+
COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
|
| 873 |
+
- COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
|
| 874 |
+
AS total_downloads
|
| 875 |
FROM base_data
|
| 876 |
+
GROUP BY model, group_key
|
| 877 |
),
|
| 878 |
|
| 879 |
+
total_downloads_cte AS (
|
| 880 |
+
SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 881 |
)
|
| 882 |
|
| 883 |
+
SELECT
|
| 884 |
+
mm.model,
|
| 885 |
+
mm.group_key,
|
| 886 |
+
mm.org_country_single,
|
| 887 |
+
mm.author,
|
| 888 |
+
mm.derived_author,
|
| 889 |
+
mm.merged_country_groups_single,
|
| 890 |
+
mm.merged_modality,
|
| 891 |
+
mm.total_downloads,
|
| 892 |
+
CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
|
| 893 |
+
FROM model_metrics mm
|
| 894 |
+
CROSS JOIN total_downloads_cte td
|
| 895 |
+
WHERE mm.total_downloads > 0
|
| 896 |
+
ORDER BY mm.total_downloads DESC
|
| 897 |
+
LIMIT {top_n * 10};
|
| 898 |
"""
|
| 899 |
|
| 900 |
return con.execute(query).fetchdf()
|
graphs/leaderboard.py
CHANGED
|
@@ -317,39 +317,59 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10, derived_author_toggl
|
|
| 317 |
Get top N entries for a leaderboard
|
| 318 |
|
| 319 |
Args:
|
| 320 |
-
filtered_df: Pandas DataFrame
|
|
|
|
|
|
|
|
|
|
| 321 |
group_col: Column to group by
|
| 322 |
top_n: Number of top entries to return
|
| 323 |
derived_author_toggle: If True, attribute to model uploader (derived_author); if False, attribute to original model creator (author)
|
| 324 |
|
| 325 |
Returns:
|
| 326 |
tuple: (display_df, download_df)
|
|
|
|
|
|
|
| 327 |
"""
|
| 328 |
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 332 |
.sum()
|
| 333 |
-
.nlargest(top_n, columns="total_downloads")
|
| 334 |
.reset_index()
|
| 335 |
-
.rename(
|
| 336 |
-
columns={
|
| 337 |
-
group_col: "Name",
|
| 338 |
-
"total_downloads": "Total Value",
|
| 339 |
-
"percent_of_total": "% of total",
|
| 340 |
-
}
|
| 341 |
-
)
|
| 342 |
)
|
| 343 |
|
| 344 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 345 |
download_top = top.copy()
|
| 346 |
download_top["Total Value"] = download_top["Total Value"].astype(int)
|
| 347 |
download_top["% of total"] = download_top["% of total"].round(2)
|
| 348 |
|
| 349 |
-
# All relevant metadata columns
|
| 350 |
meta_cols = meta_cols_map.get(group_col, [])
|
| 351 |
|
| 352 |
-
# Collect
|
| 353 |
meta_map = {}
|
| 354 |
download_map = {}
|
| 355 |
|
|
@@ -359,7 +379,7 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10, derived_author_toggl
|
|
| 359 |
download_map[name] = {}
|
| 360 |
for col in meta_cols:
|
| 361 |
if col in name_data.columns:
|
| 362 |
-
unique_vals = name_data[col].unique()
|
| 363 |
meta_map[name][col] = list(unique_vals)
|
| 364 |
download_map[name][col] = list(unique_vals)
|
| 365 |
|
|
@@ -381,7 +401,6 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10, derived_author_toggl
|
|
| 381 |
except Exception:
|
| 382 |
flag_emoji = country_emoji_fallback.get(c, "🌍")
|
| 383 |
chips.append((flag_emoji, c, "country"))
|
| 384 |
-
# Add downloads chip for country (only once)
|
| 385 |
|
| 386 |
# Author - use derived_author_toggle to determine which column
|
| 387 |
author_key = "derived_author" if derived_author_toggle else "author"
|
|
@@ -399,67 +418,66 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10, derived_author_toggl
|
|
| 399 |
if pd.notna(m):
|
| 400 |
chips.append(("", m, "modality"))
|
| 401 |
|
| 402 |
-
# Total downloads
|
| 403 |
-
|
| 404 |
-
|
|
|
|
|
|
|
|
|
|
| 405 |
chips.append(("⬇️", formatted_downloads, "downloads"))
|
| 406 |
|
| 407 |
return chips
|
| 408 |
|
| 409 |
-
#
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
download_info = {}
|
| 413 |
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
|
|
|
| 417 |
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 421 |
else:
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
top["Metadata"] = top["Name"].astype(object).apply(build_metadata)
|
| 428 |
|
| 429 |
-
# Capitalize "user" back to "User" for display
|
| 430 |
-
top["Name"] = top["Name"].replace("user", "User")
|
| 431 |
-
|
| 432 |
-
# Build download dataframe with metadata
|
| 433 |
-
download_info_list = [build_download_metadata(nm) for nm in download_top["Name"]]
|
| 434 |
download_info_df = pd.DataFrame(download_info_list)
|
| 435 |
-
download_top = pd.concat([download_top, download_info_df], axis=1)
|
| 436 |
|
| 437 |
-
return
|
| 438 |
|
| 439 |
|
| 440 |
def get_top_n_from_duckdb(
|
| 441 |
con, group_col, top_n=10, time_filter=None, view="all_downloads"
|
| 442 |
):
|
| 443 |
"""
|
| 444 |
-
Query DuckDB directly to get
|
| 445 |
-
|
| 446 |
-
Args:
|
| 447 |
-
con: DuckDB connection object
|
| 448 |
-
group_col: Column to group by
|
| 449 |
-
top_n: Number of top entries
|
| 450 |
-
time_filter: Optional tuple of (start_timestamp, end_timestamp)
|
| 451 |
-
|
| 452 |
-
Returns:
|
| 453 |
-
Pandas DataFrame with only the rows needed for top N
|
| 454 |
"""
|
| 455 |
-
#
|
| 456 |
-
|
| 457 |
-
if time_filter:
|
| 458 |
start = pd.to_datetime(time_filter[0], unit="s")
|
| 459 |
end = pd.to_datetime(time_filter[1], unit="s")
|
| 460 |
-
|
|
|
|
|
|
|
|
|
|
| 461 |
|
| 462 |
-
|
|
|
|
|
|
|
|
|
|
| 463 |
if group_col == "org_country_single":
|
| 464 |
group_expr = """CASE
|
| 465 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
|
@@ -469,108 +487,127 @@ def get_top_n_from_duckdb(
|
|
| 469 |
else:
|
| 470 |
group_expr = group_col
|
| 471 |
|
| 472 |
-
#
|
| 473 |
if group_col == "derived_author":
|
| 474 |
query = f"""
|
| 475 |
WITH base_data AS (
|
| 476 |
-
SELECT
|
| 477 |
{group_expr} AS group_key,
|
| 478 |
CASE
|
| 479 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
| 480 |
-
WHEN org_country_single IN ('International', 'Online') THEN 'International/Online'
|
| 481 |
ELSE org_country_single
|
| 482 |
END AS org_country_single,
|
| 483 |
author,
|
| 484 |
derived_author,
|
| 485 |
merged_country_groups_single,
|
| 486 |
merged_modality,
|
| 487 |
-
|
| 488 |
-
|
|
|
|
| 489 |
FROM {view}
|
| 490 |
-
{time_clause}
|
| 491 |
),
|
| 492 |
-
|
| 493 |
-
-- Create a lookup table for derived_author -> country
|
| 494 |
author_country_lookup AS (
|
| 495 |
SELECT DISTINCT
|
| 496 |
author,
|
| 497 |
-
FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY
|
| 498 |
FROM base_data
|
| 499 |
WHERE author IS NOT NULL
|
| 500 |
),
|
| 501 |
|
| 502 |
-
|
| 503 |
-
SELECT
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 504 |
FROM base_data
|
|
|
|
| 505 |
),
|
| 506 |
|
| 507 |
-
|
| 508 |
-
SELECT
|
| 509 |
-
b.group_key AS name,
|
| 510 |
-
SUM(b.downloads) AS total_downloads,
|
| 511 |
-
ROUND(SUM(b.downloads) * 100.0 / t.total_downloads_all, 2) AS percent_of_total,
|
| 512 |
-
COALESCE(acl.author_country, ANY_VALUE(b.org_country_single)) AS org_country_single,
|
| 513 |
-
ANY_VALUE(b.author) AS author,
|
| 514 |
-
ANY_VALUE(b.derived_author) AS derived_author,
|
| 515 |
-
ANY_VALUE(b.merged_country_groups_single) AS merged_country_groups_single,
|
| 516 |
-
ANY_VALUE(b.merged_modality) AS merged_modality,
|
| 517 |
-
ANY_VALUE(b.model) AS model
|
| 518 |
-
FROM base_data b
|
| 519 |
-
CROSS JOIN total_downloads_cte t
|
| 520 |
-
LEFT JOIN author_country_lookup acl ON b.group_key = acl.author
|
| 521 |
-
GROUP BY b.group_key, acl.author_country, t.total_downloads_all
|
| 522 |
)
|
| 523 |
|
| 524 |
-
SELECT
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 528 |
"""
|
| 529 |
else:
|
| 530 |
query = f"""
|
| 531 |
WITH base_data AS (
|
| 532 |
-
SELECT
|
| 533 |
{group_expr} AS group_key,
|
| 534 |
CASE
|
| 535 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
| 536 |
-
WHEN org_country_single IN ('International', 'Online') THEN 'International/Online'
|
| 537 |
ELSE org_country_single
|
| 538 |
END AS org_country_single,
|
| 539 |
author,
|
| 540 |
derived_author,
|
| 541 |
merged_country_groups_single,
|
| 542 |
merged_modality,
|
| 543 |
-
|
| 544 |
-
|
|
|
|
| 545 |
FROM {view}
|
| 546 |
-
{time_clause}
|
| 547 |
),
|
| 548 |
|
| 549 |
-
|
| 550 |
-
SELECT
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 551 |
FROM base_data
|
|
|
|
| 552 |
),
|
| 553 |
|
| 554 |
-
|
| 555 |
-
SELECT
|
| 556 |
-
b.group_key AS name,
|
| 557 |
-
SUM(b.downloads) AS total_downloads,
|
| 558 |
-
ROUND(SUM(b.downloads) * 100.0 / t.total_downloads_all, 2) AS percent_of_total,
|
| 559 |
-
ANY_VALUE(b.org_country_single) AS org_country_single,
|
| 560 |
-
ANY_VALUE(b.author) AS author,
|
| 561 |
-
ANY_VALUE(b.derived_author) AS derived_author,
|
| 562 |
-
ANY_VALUE(b.merged_country_groups_single) AS merged_country_groups_single,
|
| 563 |
-
ANY_VALUE(b.merged_modality) AS merged_modality,
|
| 564 |
-
ANY_VALUE(b.model) AS model
|
| 565 |
-
FROM base_data b
|
| 566 |
-
CROSS JOIN total_downloads_cte t
|
| 567 |
-
GROUP BY b.group_key, t.total_downloads_all
|
| 568 |
)
|
| 569 |
|
| 570 |
-
SELECT
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 574 |
"""
|
| 575 |
|
| 576 |
try:
|
|
|
|
| 317 |
Get top N entries for a leaderboard
|
| 318 |
|
| 319 |
Args:
|
| 320 |
+
filtered_df: Pandas DataFrame of model-level rows. Must contain:
|
| 321 |
+
- group_col (the grouping key)
|
| 322 |
+
- total_downloads (per-model downloads for the requested window)
|
| 323 |
+
- plus metadata columns: org_country_single, author, derived_author, merged_country_groups_single, merged_modality, model
|
| 324 |
group_col: Column to group by
|
| 325 |
top_n: Number of top entries to return
|
| 326 |
derived_author_toggle: If True, attribute to model uploader (derived_author); if False, attribute to original model creator (author)
|
| 327 |
|
| 328 |
Returns:
|
| 329 |
tuple: (display_df, download_df)
|
| 330 |
+
display_df: DataFrame with columns ["Name","Metadata","% of total"] for rendering
|
| 331 |
+
download_df: DataFrame suitable for CSV download with numeric totals and metadata columns
|
| 332 |
"""
|
| 333 |
|
| 334 |
+
if filtered_df is None or filtered_df.empty:
|
| 335 |
+
return pd.DataFrame(), pd.DataFrame()
|
| 336 |
+
|
| 337 |
+
# Ensure numeric total_downloads
|
| 338 |
+
if "total_downloads" not in filtered_df.columns:
|
| 339 |
+
# fallback if older code still returned 'downloads' (unlikely)
|
| 340 |
+
if "downloads" in filtered_df.columns:
|
| 341 |
+
filtered_df["total_downloads"] = filtered_df["downloads"]
|
| 342 |
+
else:
|
| 343 |
+
filtered_df["total_downloads"] = 0
|
| 344 |
+
|
| 345 |
+
# Compute overall total across all models in this filtered set
|
| 346 |
+
total_all = filtered_df["total_downloads"].sum()
|
| 347 |
+
if total_all == 0:
|
| 348 |
+
return pd.DataFrame(), pd.DataFrame()
|
| 349 |
+
|
| 350 |
+
# Sum per group (group_col) to get group totals
|
| 351 |
+
grouped = (
|
| 352 |
+
filtered_df.groupby(group_col)["total_downloads"]
|
| 353 |
.sum()
|
|
|
|
| 354 |
.reset_index()
|
| 355 |
+
.rename(columns={group_col: "Name", "total_downloads": "Total Value"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 356 |
)
|
| 357 |
|
| 358 |
+
# Pick top N groups by summed downloads
|
| 359 |
+
top = grouped.nlargest(top_n, columns="Total Value").reset_index(drop=True)
|
| 360 |
+
|
| 361 |
+
# Compute percent of total for display (rounded)
|
| 362 |
+
top["% of total"] = top["Total Value"].apply(lambda v: round(v * 100.0 / total_all, 2))
|
| 363 |
+
|
| 364 |
+
# Build download version (numeric)
|
| 365 |
download_top = top.copy()
|
| 366 |
download_top["Total Value"] = download_top["Total Value"].astype(int)
|
| 367 |
download_top["% of total"] = download_top["% of total"].round(2)
|
| 368 |
|
| 369 |
+
# All relevant metadata columns for the grouping
|
| 370 |
meta_cols = meta_cols_map.get(group_col, [])
|
| 371 |
|
| 372 |
+
# Collect metadata per group by inspecting the underlying model-level rows
|
| 373 |
meta_map = {}
|
| 374 |
download_map = {}
|
| 375 |
|
|
|
|
| 379 |
download_map[name] = {}
|
| 380 |
for col in meta_cols:
|
| 381 |
if col in name_data.columns:
|
| 382 |
+
unique_vals = name_data[col].dropna().unique()
|
| 383 |
meta_map[name][col] = list(unique_vals)
|
| 384 |
download_map[name][col] = list(unique_vals)
|
| 385 |
|
|
|
|
| 401 |
except Exception:
|
| 402 |
flag_emoji = country_emoji_fallback.get(c, "🌍")
|
| 403 |
chips.append((flag_emoji, c, "country"))
|
|
|
|
| 404 |
|
| 405 |
# Author - use derived_author_toggle to determine which column
|
| 406 |
author_key = "derived_author" if derived_author_toggle else "author"
|
|
|
|
| 418 |
if pd.notna(m):
|
| 419 |
chips.append(("", m, "modality"))
|
| 420 |
|
| 421 |
+
# Total downloads (aggregate numeric value for this group)
|
| 422 |
+
# Use the summed value from top (we can retrieve it)
|
| 423 |
+
# but we also include any per-model totals if desired - keep simple: use group total
|
| 424 |
+
group_total = int(top.loc[top["Name"] == nm, "Total Value"].iloc[0]) if nm in top["Name"].values else None
|
| 425 |
+
if group_total is not None:
|
| 426 |
+
formatted_downloads = format_large_number(group_total)
|
| 427 |
chips.append(("⬇️", formatted_downloads, "downloads"))
|
| 428 |
|
| 429 |
return chips
|
| 430 |
|
| 431 |
+
# Attach Metadata column for display DataFrame
|
| 432 |
+
display_df = top.rename(columns={"Total Value": "total_downloads"})
|
| 433 |
+
display_df["Metadata"] = display_df["Name"].astype(object).apply(build_metadata)
|
|
|
|
| 434 |
|
| 435 |
+
# Format display_df columns for render_table_content
|
| 436 |
+
display_df_formatted = display_df.rename(columns={"% of total": "% of total"})
|
| 437 |
+
# Keep only necessary columns in expected order
|
| 438 |
+
display_for_render = display_df_formatted[["Name", "Metadata", "% of total"]]
|
| 439 |
|
| 440 |
+
# Build download dataframe with metadata for CSV
|
| 441 |
+
download_info_list = []
|
| 442 |
+
for nm in download_top["Name"]:
|
| 443 |
+
info = {}
|
| 444 |
+
meta = download_map.get(nm, {})
|
| 445 |
+
for col in meta_cols:
|
| 446 |
+
if col in meta and meta[col]:
|
| 447 |
+
info[col] = ", ".join(str(v) for v in meta[col] if pd.notna(v))
|
| 448 |
else:
|
| 449 |
+
info[col] = ""
|
| 450 |
+
# attach totals
|
| 451 |
+
info["Total Value"] = int(download_top.loc[download_top["Name"] == nm, "Total Value"].iloc[0])
|
| 452 |
+
info["% of total"] = float(download_top.loc[download_top["Name"] == nm, "% of total"].iloc[0])
|
| 453 |
+
download_info_list.append(info)
|
|
|
|
| 454 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 455 |
download_info_df = pd.DataFrame(download_info_list)
|
| 456 |
+
download_top = pd.concat([download_top.reset_index(drop=True), download_info_df.reset_index(drop=True)], axis=1)
|
| 457 |
|
| 458 |
+
return display_for_render, download_top
|
| 459 |
|
| 460 |
|
| 461 |
def get_top_n_from_duckdb(
|
| 462 |
con, group_col, top_n=10, time_filter=None, view="all_downloads"
|
| 463 |
):
|
| 464 |
"""
|
| 465 |
+
Query DuckDB directly to get model-level rows with per-model total_downloads (delta or full)
|
| 466 |
+
Returns rows similar to _get_filtered_top_n_from_duckdb in app.py.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 467 |
"""
|
| 468 |
+
# Compute date window
|
| 469 |
+
if time_filter and len(time_filter) == 2:
|
|
|
|
| 470 |
start = pd.to_datetime(time_filter[0], unit="s")
|
| 471 |
end = pd.to_datetime(time_filter[1], unit="s")
|
| 472 |
+
else:
|
| 473 |
+
start = pd.to_datetime("1970-01-01")
|
| 474 |
+
# We cannot access end_dt here; rely on time_filter for end in typical use.
|
| 475 |
+
end = pd.Timestamp.now()
|
| 476 |
|
| 477 |
+
start_str = str(start)
|
| 478 |
+
end_str = str(end)
|
| 479 |
+
|
| 480 |
+
# If grouping by country, transform some country values
|
| 481 |
if group_col == "org_country_single":
|
| 482 |
group_expr = """CASE
|
| 483 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
|
|
|
| 487 |
else:
|
| 488 |
group_expr = group_col
|
| 489 |
|
| 490 |
+
# Derived author special-case
|
| 491 |
if group_col == "derived_author":
|
| 492 |
query = f"""
|
| 493 |
WITH base_data AS (
|
| 494 |
+
SELECT
|
| 495 |
{group_expr} AS group_key,
|
| 496 |
CASE
|
| 497 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
| 498 |
+
WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
|
| 499 |
ELSE org_country_single
|
| 500 |
END AS org_country_single,
|
| 501 |
author,
|
| 502 |
derived_author,
|
| 503 |
merged_country_groups_single,
|
| 504 |
merged_modality,
|
| 505 |
+
model,
|
| 506 |
+
time,
|
| 507 |
+
downloadsAllTime
|
| 508 |
FROM {view}
|
|
|
|
| 509 |
),
|
| 510 |
+
|
|
|
|
| 511 |
author_country_lookup AS (
|
| 512 |
SELECT DISTINCT
|
| 513 |
author,
|
| 514 |
+
FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloadsAllTime DESC) AS author_country
|
| 515 |
FROM base_data
|
| 516 |
WHERE author IS NOT NULL
|
| 517 |
),
|
| 518 |
|
| 519 |
+
model_metrics AS (
|
| 520 |
+
SELECT
|
| 521 |
+
model,
|
| 522 |
+
group_key,
|
| 523 |
+
ANY_VALUE(org_country_single) AS org_country_single,
|
| 524 |
+
ANY_VALUE(author) AS author,
|
| 525 |
+
ANY_VALUE(derived_author) AS derived_author,
|
| 526 |
+
ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
|
| 527 |
+
ANY_VALUE(merged_modality) AS merged_modality,
|
| 528 |
+
COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
|
| 529 |
+
- COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
|
| 530 |
+
AS total_downloads
|
| 531 |
FROM base_data
|
| 532 |
+
GROUP BY model, group_key
|
| 533 |
),
|
| 534 |
|
| 535 |
+
total_downloads_cte AS (
|
| 536 |
+
SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 537 |
)
|
| 538 |
|
| 539 |
+
SELECT
|
| 540 |
+
mm.model,
|
| 541 |
+
mm.group_key,
|
| 542 |
+
COALESCE(acl.author_country, mm.org_country_single) AS org_country_single,
|
| 543 |
+
mm.author,
|
| 544 |
+
mm.derived_author,
|
| 545 |
+
mm.merged_country_groups_single,
|
| 546 |
+
mm.merged_modality,
|
| 547 |
+
mm.total_downloads,
|
| 548 |
+
CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
|
| 549 |
+
FROM model_metrics mm
|
| 550 |
+
LEFT JOIN author_country_lookup acl ON mm.group_key = acl.author
|
| 551 |
+
CROSS JOIN total_downloads_cte td
|
| 552 |
+
WHERE mm.total_downloads > 0
|
| 553 |
+
ORDER BY mm.total_downloads DESC
|
| 554 |
+
LIMIT {top_n * 10};
|
| 555 |
"""
|
| 556 |
else:
|
| 557 |
query = f"""
|
| 558 |
WITH base_data AS (
|
| 559 |
+
SELECT
|
| 560 |
{group_expr} AS group_key,
|
| 561 |
CASE
|
| 562 |
WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
|
| 563 |
+
WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
|
| 564 |
ELSE org_country_single
|
| 565 |
END AS org_country_single,
|
| 566 |
author,
|
| 567 |
derived_author,
|
| 568 |
merged_country_groups_single,
|
| 569 |
merged_modality,
|
| 570 |
+
model,
|
| 571 |
+
time,
|
| 572 |
+
downloadsAllTime
|
| 573 |
FROM {view}
|
|
|
|
| 574 |
),
|
| 575 |
|
| 576 |
+
model_metrics AS (
|
| 577 |
+
SELECT
|
| 578 |
+
model,
|
| 579 |
+
group_key,
|
| 580 |
+
ANY_VALUE(org_country_single) AS org_country_single,
|
| 581 |
+
ANY_VALUE(author) AS author,
|
| 582 |
+
ANY_VALUE(derived_author) AS derived_author,
|
| 583 |
+
ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
|
| 584 |
+
ANY_VALUE(merged_modality) AS merged_modality,
|
| 585 |
+
COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
|
| 586 |
+
- COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
|
| 587 |
+
AS total_downloads
|
| 588 |
FROM base_data
|
| 589 |
+
GROUP BY model, group_key
|
| 590 |
),
|
| 591 |
|
| 592 |
+
total_downloads_cte AS (
|
| 593 |
+
SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 594 |
)
|
| 595 |
|
| 596 |
+
SELECT
|
| 597 |
+
mm.model,
|
| 598 |
+
mm.group_key,
|
| 599 |
+
mm.org_country_single,
|
| 600 |
+
mm.author,
|
| 601 |
+
mm.derived_author,
|
| 602 |
+
mm.merged_country_groups_single,
|
| 603 |
+
mm.merged_modality,
|
| 604 |
+
mm.total_downloads,
|
| 605 |
+
CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
|
| 606 |
+
FROM model_metrics mm
|
| 607 |
+
CROSS JOIN total_downloads_cte td
|
| 608 |
+
WHERE mm.total_downloads > 0
|
| 609 |
+
ORDER BY mm.total_downloads DESC
|
| 610 |
+
LIMIT {top_n * 10};
|
| 611 |
"""
|
| 612 |
|
| 613 |
try:
|