emsesc commited on
Commit
fddfe1e
·
1 Parent(s): a8701cd

temp migration

Browse files
Files changed (2) hide show
  1. app.py +90 -63
  2. graphs/leaderboard.py +150 -113
app.py CHANGED
@@ -59,8 +59,8 @@ con.execute("SET enable_object_cache = false;")
59
 
60
  # Load parquet files from Hugging Face using DuckDB
61
  HF_DATASET_ID = "mmpr/open_model_evolution_data"
62
- hf_parquet_url_1 = "https://huggingface.co/datasets/mmpr/open_model_evolution_data/resolve/main/all_downloads_with_annotations.parquet"
63
- hf_parquet_url_2 = "https://huggingface.co/datasets/mmpr/open_model_evolution_data/resolve/main/one_year_rolling.parquet"
64
 
65
  print(f"Attempting to connect to dataset from Hugging Face Hub: {HF_DATASET_ID}")
66
  try:
@@ -745,17 +745,26 @@ def _get_filtered_top_n_from_duckdb(
745
  slider_value, group_col, top_n, view="all_downloads"
746
  ):
747
  """
748
- Query DuckDB directly to get top N entries with metadata
749
- This minimizes data transfer by doing aggregation in DuckDB
 
 
 
 
750
  """
751
- # Build time filter clause
752
- time_clause = ""
753
  if slider_value and len(slider_value) == 2:
754
  start = pd.to_datetime(slider_value[0], unit="s")
755
  end = pd.to_datetime(slider_value[1], unit="s")
756
- time_clause = f"WHERE time >= '{start}' AND time <= '{end}'"
 
 
 
 
 
757
 
758
- # If grouping by country, group by the transformed country column
759
  if group_col == "org_country_single":
760
  group_expr = """CASE
761
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
@@ -765,12 +774,11 @@ def _get_filtered_top_n_from_duckdb(
765
  else:
766
  group_expr = group_col
767
 
768
- # Build a lookup for author -> country mapping
769
- # When grouping by derived_author, we need to find the country where derived_author = author
770
  if group_col == "derived_author":
771
  query = f"""
772
  WITH base_data AS (
773
- SELECT
774
  {group_expr} AS group_key,
775
  CASE
776
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
@@ -781,93 +789,112 @@ def _get_filtered_top_n_from_duckdb(
781
  derived_author,
782
  merged_country_groups_single,
783
  merged_modality,
784
- downloads,
785
- model
 
786
  FROM {view}
787
- {time_clause}
788
  ),
789
-
790
- -- Create a lookup table for derived_author -> country
791
  author_country_lookup AS (
792
  SELECT DISTINCT
793
  author,
794
- FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloads DESC) AS author_country
795
  FROM base_data
796
  WHERE author IS NOT NULL
797
  ),
798
 
799
- total_downloads_cte AS (
800
- SELECT SUM(downloads) AS total_downloads_all
 
 
 
 
 
 
 
 
 
 
801
  FROM base_data
 
802
  ),
803
 
804
- top_items AS (
805
- SELECT
806
- b.group_key AS name,
807
- SUM(b.downloads) AS total_downloads,
808
- ROUND(SUM(b.downloads) * 100.0 / t.total_downloads_all, 2) AS percent_of_total,
809
- COALESCE(acl.author_country, ANY_VALUE(b.org_country_single)) AS org_country_single,
810
- ANY_VALUE(b.author) AS author,
811
- ANY_VALUE(b.derived_author) AS derived_author,
812
- ANY_VALUE(b.merged_country_groups_single) AS merged_country_groups_single,
813
- ANY_VALUE(b.merged_modality) AS merged_modality,
814
- ANY_VALUE(b.model) AS model
815
- FROM base_data b
816
- CROSS JOIN total_downloads_cte t
817
- LEFT JOIN author_country_lookup acl ON b.group_key = acl.author
818
- GROUP BY b.group_key, acl.author_country, t.total_downloads_all
819
  )
820
 
821
- SELECT *
822
- FROM top_items
823
- ORDER BY total_downloads DESC
824
- LIMIT {top_n};
 
 
 
 
 
 
 
 
 
 
 
 
825
  """
826
  else:
827
  query = f"""
828
  WITH base_data AS (
829
- SELECT
830
  {group_expr} AS group_key,
831
  CASE
832
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
833
- WHEN org_country_single IN ('International', 'Online') THEN 'International/Online'
834
  ELSE org_country_single
835
  END AS org_country_single,
836
  author,
837
  derived_author,
838
  merged_country_groups_single,
839
  merged_modality,
840
- downloads,
841
- model
 
842
  FROM {view}
843
- {time_clause}
844
  ),
845
 
846
- total_downloads_cte AS (
847
- SELECT SUM(downloads) AS total_downloads_all
 
 
 
 
 
 
 
 
 
 
848
  FROM base_data
 
849
  ),
850
 
851
- top_items AS (
852
- SELECT
853
- b.group_key AS name,
854
- SUM(b.downloads) AS total_downloads,
855
- ROUND(SUM(b.downloads) * 100.0 / t.total_downloads_all, 2) AS percent_of_total,
856
- ANY_VALUE(b.org_country_single) AS org_country_single,
857
- ANY_VALUE(b.author) AS author,
858
- ANY_VALUE(b.derived_author) AS derived_author,
859
- ANY_VALUE(b.merged_country_groups_single) AS merged_country_groups_single,
860
- ANY_VALUE(b.merged_modality) AS merged_modality,
861
- ANY_VALUE(b.model) AS model
862
- FROM base_data b
863
- CROSS JOIN total_downloads_cte t
864
- GROUP BY b.group_key, t.total_downloads_all
865
  )
866
 
867
- SELECT *
868
- FROM top_items
869
- ORDER BY total_downloads DESC
870
- LIMIT {top_n};
 
 
 
 
 
 
 
 
 
 
 
871
  """
872
 
873
  return con.execute(query).fetchdf()
 
59
 
60
  # Load parquet files from Hugging Face using DuckDB
61
  HF_DATASET_ID = "mmpr/open_model_evolution_data"
62
+ hf_parquet_url_1 = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/all_downloads_with_annotations-1.parquet"
63
+ hf_parquet_url_2 = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/one_year_rolling-2.parquet"
64
 
65
  print(f"Attempting to connect to dataset from Hugging Face Hub: {HF_DATASET_ID}")
66
  try:
 
745
  slider_value, group_col, top_n, view="all_downloads"
746
  ):
747
  """
748
+ Query DuckDB to get model-level rows with per-model total_downloads (delta or full)
749
+ Returns a DataFrame with columns including:
750
+ - group_key (the grouping column)
751
+ - org_country_single, author, derived_author, merged_country_groups_single, merged_modality, model
752
+ - total_downloads (per-model downloads in requested window)
753
+ - percent_of_total (percent of total across all returned model deltas)
754
  """
755
+
756
+ # Compute date window (if slider_value provided, use it; otherwise cover full range)
757
  if slider_value and len(slider_value) == 2:
758
  start = pd.to_datetime(slider_value[0], unit="s")
759
  end = pd.to_datetime(slider_value[1], unit="s")
760
+ else:
761
+ start = pd.to_datetime("1970-01-01")
762
+ end = end_dt # defined near top of file when parquet was loaded
763
+
764
+ start_str = str(start)
765
+ end_str = str(end)
766
 
767
+ # If grouping by country, transform some country values
768
  if group_col == "org_country_single":
769
  group_expr = """CASE
770
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
 
774
  else:
775
  group_expr = group_col
776
 
777
+ # Derived-author requires author->country lookup; build separate SQL for that case
 
778
  if group_col == "derived_author":
779
  query = f"""
780
  WITH base_data AS (
781
+ SELECT
782
  {group_expr} AS group_key,
783
  CASE
784
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
 
789
  derived_author,
790
  merged_country_groups_single,
791
  merged_modality,
792
+ model,
793
+ time,
794
+ downloadsAllTime
795
  FROM {view}
 
796
  ),
797
+
 
798
  author_country_lookup AS (
799
  SELECT DISTINCT
800
  author,
801
+ FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloadsAllTime DESC) AS author_country
802
  FROM base_data
803
  WHERE author IS NOT NULL
804
  ),
805
 
806
+ model_metrics AS (
807
+ SELECT
808
+ model,
809
+ group_key,
810
+ ANY_VALUE(org_country_single) AS org_country_single,
811
+ ANY_VALUE(author) AS author,
812
+ ANY_VALUE(derived_author) AS derived_author,
813
+ ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
814
+ ANY_VALUE(merged_modality) AS merged_modality,
815
+ COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
816
+ - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
817
+ AS total_downloads
818
  FROM base_data
819
+ GROUP BY model, group_key
820
  ),
821
 
822
+ total_downloads_cte AS (
823
+ SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
 
 
 
 
 
 
 
 
 
 
 
 
 
824
  )
825
 
826
+ SELECT
827
+ mm.model,
828
+ mm.group_key,
829
+ COALESCE(acl.author_country, mm.org_country_single) AS org_country_single,
830
+ mm.author,
831
+ mm.derived_author,
832
+ mm.merged_country_groups_single,
833
+ mm.merged_modality,
834
+ mm.total_downloads,
835
+ CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
836
+ FROM model_metrics mm
837
+ LEFT JOIN author_country_lookup acl ON mm.group_key = acl.author
838
+ CROSS JOIN total_downloads_cte td
839
+ WHERE mm.total_downloads > 0
840
+ ORDER BY mm.total_downloads DESC
841
+ LIMIT {top_n * 10};
842
  """
843
  else:
844
  query = f"""
845
  WITH base_data AS (
846
+ SELECT
847
  {group_expr} AS group_key,
848
  CASE
849
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
850
+ WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
851
  ELSE org_country_single
852
  END AS org_country_single,
853
  author,
854
  derived_author,
855
  merged_country_groups_single,
856
  merged_modality,
857
+ model,
858
+ time,
859
+ downloadsAllTime
860
  FROM {view}
 
861
  ),
862
 
863
+ model_metrics AS (
864
+ SELECT
865
+ model,
866
+ group_key,
867
+ ANY_VALUE(org_country_single) AS org_country_single,
868
+ ANY_VALUE(author) AS author,
869
+ ANY_VALUE(derived_author) AS derived_author,
870
+ ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
871
+ ANY_VALUE(merged_modality) AS merged_modality,
872
+ COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
873
+ - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
874
+ AS total_downloads
875
  FROM base_data
876
+ GROUP BY model, group_key
877
  ),
878
 
879
+ total_downloads_cte AS (
880
+ SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
 
 
 
 
 
 
 
 
 
 
 
 
881
  )
882
 
883
+ SELECT
884
+ mm.model,
885
+ mm.group_key,
886
+ mm.org_country_single,
887
+ mm.author,
888
+ mm.derived_author,
889
+ mm.merged_country_groups_single,
890
+ mm.merged_modality,
891
+ mm.total_downloads,
892
+ CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
893
+ FROM model_metrics mm
894
+ CROSS JOIN total_downloads_cte td
895
+ WHERE mm.total_downloads > 0
896
+ ORDER BY mm.total_downloads DESC
897
+ LIMIT {top_n * 10};
898
  """
899
 
900
  return con.execute(query).fetchdf()
graphs/leaderboard.py CHANGED
@@ -317,39 +317,59 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10, derived_author_toggl
317
  Get top N entries for a leaderboard
318
 
319
  Args:
320
- filtered_df: Pandas DataFrame (already filtered by time from DuckDB query)
 
 
 
321
  group_col: Column to group by
322
  top_n: Number of top entries to return
323
  derived_author_toggle: If True, attribute to model uploader (derived_author); if False, attribute to original model creator (author)
324
 
325
  Returns:
326
  tuple: (display_df, download_df)
 
 
327
  """
328
 
329
- # Group by and get top N
330
- top = (
331
- filtered_df.groupby(group_col)[["total_downloads", "percent_of_total"]]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
332
  .sum()
333
- .nlargest(top_n, columns="total_downloads")
334
  .reset_index()
335
- .rename(
336
- columns={
337
- group_col: "Name",
338
- "total_downloads": "Total Value",
339
- "percent_of_total": "% of total",
340
- }
341
- )
342
  )
343
 
344
- # Create a downloadable version of the leaderboard
 
 
 
 
 
 
345
  download_top = top.copy()
346
  download_top["Total Value"] = download_top["Total Value"].astype(int)
347
  download_top["% of total"] = download_top["% of total"].round(2)
348
 
349
- # All relevant metadata columns
350
  meta_cols = meta_cols_map.get(group_col, [])
351
 
352
- # Collect all metadata per top n for each category (country, author, model)
353
  meta_map = {}
354
  download_map = {}
355
 
@@ -359,7 +379,7 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10, derived_author_toggl
359
  download_map[name] = {}
360
  for col in meta_cols:
361
  if col in name_data.columns:
362
- unique_vals = name_data[col].unique()
363
  meta_map[name][col] = list(unique_vals)
364
  download_map[name][col] = list(unique_vals)
365
 
@@ -381,7 +401,6 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10, derived_author_toggl
381
  except Exception:
382
  flag_emoji = country_emoji_fallback.get(c, "🌍")
383
  chips.append((flag_emoji, c, "country"))
384
- # Add downloads chip for country (only once)
385
 
386
  # Author - use derived_author_toggle to determine which column
387
  author_key = "derived_author" if derived_author_toggle else "author"
@@ -399,67 +418,66 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10, derived_author_toggl
399
  if pd.notna(m):
400
  chips.append(("", m, "modality"))
401
 
402
- # Total downloads
403
- for d in meta.get("total_downloads", []):
404
- formatted_downloads = format_large_number(d)
 
 
 
405
  chips.append(("⬇️", formatted_downloads, "downloads"))
406
 
407
  return chips
408
 
409
- # Function to create downloadable dataframe metadata
410
- def build_download_metadata(nm):
411
- meta = download_map.get(nm, {})
412
- download_info = {}
413
 
414
- for col in meta_cols:
415
- if col not in meta or not meta[col]:
416
- continue
 
417
 
418
- vals = meta.get(col, [])
419
- if vals:
420
- download_info[col] = ", ".join(str(v) for v in vals if pd.notna(v))
 
 
 
 
 
421
  else:
422
- download_info[col] = ""
423
-
424
- return download_info
425
-
426
- # Apply metadata builder to top dataframe
427
- top["Metadata"] = top["Name"].astype(object).apply(build_metadata)
428
 
429
- # Capitalize "user" back to "User" for display
430
- top["Name"] = top["Name"].replace("user", "User")
431
-
432
- # Build download dataframe with metadata
433
- download_info_list = [build_download_metadata(nm) for nm in download_top["Name"]]
434
  download_info_df = pd.DataFrame(download_info_list)
435
- download_top = pd.concat([download_top, download_info_df], axis=1)
436
 
437
- return top[["Name", "Metadata", "% of total"]], download_top
438
 
439
 
440
  def get_top_n_from_duckdb(
441
  con, group_col, top_n=10, time_filter=None, view="all_downloads"
442
  ):
443
  """
444
- Query DuckDB directly to get top N entries with minimal data transfer
445
-
446
- Args:
447
- con: DuckDB connection object
448
- group_col: Column to group by
449
- top_n: Number of top entries
450
- time_filter: Optional tuple of (start_timestamp, end_timestamp)
451
-
452
- Returns:
453
- Pandas DataFrame with only the rows needed for top N
454
  """
455
- # Build time filter clause
456
- time_clause = ""
457
- if time_filter:
458
  start = pd.to_datetime(time_filter[0], unit="s")
459
  end = pd.to_datetime(time_filter[1], unit="s")
460
- time_clause = f"WHERE time >= '{start}' AND time <= '{end}'"
 
 
 
461
 
462
- # If grouping by country, group by the transformed country column
 
 
 
463
  if group_col == "org_country_single":
464
  group_expr = """CASE
465
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
@@ -469,108 +487,127 @@ def get_top_n_from_duckdb(
469
  else:
470
  group_expr = group_col
471
 
472
- # When grouping by derived_author, lookup the country where derived_author = author
473
  if group_col == "derived_author":
474
  query = f"""
475
  WITH base_data AS (
476
- SELECT
477
  {group_expr} AS group_key,
478
  CASE
479
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
480
- WHEN org_country_single IN ('International', 'Online') THEN 'International/Online'
481
  ELSE org_country_single
482
  END AS org_country_single,
483
  author,
484
  derived_author,
485
  merged_country_groups_single,
486
  merged_modality,
487
- downloads,
488
- model
 
489
  FROM {view}
490
- {time_clause}
491
  ),
492
-
493
- -- Create a lookup table for derived_author -> country
494
  author_country_lookup AS (
495
  SELECT DISTINCT
496
  author,
497
- FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloads DESC) AS author_country
498
  FROM base_data
499
  WHERE author IS NOT NULL
500
  ),
501
 
502
- total_downloads_cte AS (
503
- SELECT SUM(downloads) AS total_downloads_all
 
 
 
 
 
 
 
 
 
 
504
  FROM base_data
 
505
  ),
506
 
507
- top_items AS (
508
- SELECT
509
- b.group_key AS name,
510
- SUM(b.downloads) AS total_downloads,
511
- ROUND(SUM(b.downloads) * 100.0 / t.total_downloads_all, 2) AS percent_of_total,
512
- COALESCE(acl.author_country, ANY_VALUE(b.org_country_single)) AS org_country_single,
513
- ANY_VALUE(b.author) AS author,
514
- ANY_VALUE(b.derived_author) AS derived_author,
515
- ANY_VALUE(b.merged_country_groups_single) AS merged_country_groups_single,
516
- ANY_VALUE(b.merged_modality) AS merged_modality,
517
- ANY_VALUE(b.model) AS model
518
- FROM base_data b
519
- CROSS JOIN total_downloads_cte t
520
- LEFT JOIN author_country_lookup acl ON b.group_key = acl.author
521
- GROUP BY b.group_key, acl.author_country, t.total_downloads_all
522
  )
523
 
524
- SELECT *
525
- FROM top_items
526
- ORDER BY total_downloads DESC
527
- LIMIT {top_n};
 
 
 
 
 
 
 
 
 
 
 
 
528
  """
529
  else:
530
  query = f"""
531
  WITH base_data AS (
532
- SELECT
533
  {group_expr} AS group_key,
534
  CASE
535
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
536
- WHEN org_country_single IN ('International', 'Online') THEN 'International/Online'
537
  ELSE org_country_single
538
  END AS org_country_single,
539
  author,
540
  derived_author,
541
  merged_country_groups_single,
542
  merged_modality,
543
- downloads,
544
- model
 
545
  FROM {view}
546
- {time_clause}
547
  ),
548
 
549
- total_downloads_cte AS (
550
- SELECT SUM(downloads) AS total_downloads_all
 
 
 
 
 
 
 
 
 
 
551
  FROM base_data
 
552
  ),
553
 
554
- top_items AS (
555
- SELECT
556
- b.group_key AS name,
557
- SUM(b.downloads) AS total_downloads,
558
- ROUND(SUM(b.downloads) * 100.0 / t.total_downloads_all, 2) AS percent_of_total,
559
- ANY_VALUE(b.org_country_single) AS org_country_single,
560
- ANY_VALUE(b.author) AS author,
561
- ANY_VALUE(b.derived_author) AS derived_author,
562
- ANY_VALUE(b.merged_country_groups_single) AS merged_country_groups_single,
563
- ANY_VALUE(b.merged_modality) AS merged_modality,
564
- ANY_VALUE(b.model) AS model
565
- FROM base_data b
566
- CROSS JOIN total_downloads_cte t
567
- GROUP BY b.group_key, t.total_downloads_all
568
  )
569
 
570
- SELECT *
571
- FROM top_items
572
- ORDER BY total_downloads DESC
573
- LIMIT {top_n};
 
 
 
 
 
 
 
 
 
 
 
574
  """
575
 
576
  try:
 
317
  Get top N entries for a leaderboard
318
 
319
  Args:
320
+ filtered_df: Pandas DataFrame of model-level rows. Must contain:
321
+ - group_col (the grouping key)
322
+ - total_downloads (per-model downloads for the requested window)
323
+ - plus metadata columns: org_country_single, author, derived_author, merged_country_groups_single, merged_modality, model
324
  group_col: Column to group by
325
  top_n: Number of top entries to return
326
  derived_author_toggle: If True, attribute to model uploader (derived_author); if False, attribute to original model creator (author)
327
 
328
  Returns:
329
  tuple: (display_df, download_df)
330
+ display_df: DataFrame with columns ["Name","Metadata","% of total"] for rendering
331
+ download_df: DataFrame suitable for CSV download with numeric totals and metadata columns
332
  """
333
 
334
+ if filtered_df is None or filtered_df.empty:
335
+ return pd.DataFrame(), pd.DataFrame()
336
+
337
+ # Ensure numeric total_downloads
338
+ if "total_downloads" not in filtered_df.columns:
339
+ # fallback if older code still returned 'downloads' (unlikely)
340
+ if "downloads" in filtered_df.columns:
341
+ filtered_df["total_downloads"] = filtered_df["downloads"]
342
+ else:
343
+ filtered_df["total_downloads"] = 0
344
+
345
+ # Compute overall total across all models in this filtered set
346
+ total_all = filtered_df["total_downloads"].sum()
347
+ if total_all == 0:
348
+ return pd.DataFrame(), pd.DataFrame()
349
+
350
+ # Sum per group (group_col) to get group totals
351
+ grouped = (
352
+ filtered_df.groupby(group_col)["total_downloads"]
353
  .sum()
 
354
  .reset_index()
355
+ .rename(columns={group_col: "Name", "total_downloads": "Total Value"})
 
 
 
 
 
 
356
  )
357
 
358
+ # Pick top N groups by summed downloads
359
+ top = grouped.nlargest(top_n, columns="Total Value").reset_index(drop=True)
360
+
361
+ # Compute percent of total for display (rounded)
362
+ top["% of total"] = top["Total Value"].apply(lambda v: round(v * 100.0 / total_all, 2))
363
+
364
+ # Build download version (numeric)
365
  download_top = top.copy()
366
  download_top["Total Value"] = download_top["Total Value"].astype(int)
367
  download_top["% of total"] = download_top["% of total"].round(2)
368
 
369
+ # All relevant metadata columns for the grouping
370
  meta_cols = meta_cols_map.get(group_col, [])
371
 
372
+ # Collect metadata per group by inspecting the underlying model-level rows
373
  meta_map = {}
374
  download_map = {}
375
 
 
379
  download_map[name] = {}
380
  for col in meta_cols:
381
  if col in name_data.columns:
382
+ unique_vals = name_data[col].dropna().unique()
383
  meta_map[name][col] = list(unique_vals)
384
  download_map[name][col] = list(unique_vals)
385
 
 
401
  except Exception:
402
  flag_emoji = country_emoji_fallback.get(c, "🌍")
403
  chips.append((flag_emoji, c, "country"))
 
404
 
405
  # Author - use derived_author_toggle to determine which column
406
  author_key = "derived_author" if derived_author_toggle else "author"
 
418
  if pd.notna(m):
419
  chips.append(("", m, "modality"))
420
 
421
+ # Total downloads (aggregate numeric value for this group)
422
+ # Use the summed value from top (we can retrieve it)
423
+ # but we also include any per-model totals if desired - keep simple: use group total
424
+ group_total = int(top.loc[top["Name"] == nm, "Total Value"].iloc[0]) if nm in top["Name"].values else None
425
+ if group_total is not None:
426
+ formatted_downloads = format_large_number(group_total)
427
  chips.append(("⬇️", formatted_downloads, "downloads"))
428
 
429
  return chips
430
 
431
+ # Attach Metadata column for display DataFrame
432
+ display_df = top.rename(columns={"Total Value": "total_downloads"})
433
+ display_df["Metadata"] = display_df["Name"].astype(object).apply(build_metadata)
 
434
 
435
+ # Format display_df columns for render_table_content
436
+ display_df_formatted = display_df.rename(columns={"% of total": "% of total"})
437
+ # Keep only necessary columns in expected order
438
+ display_for_render = display_df_formatted[["Name", "Metadata", "% of total"]]
439
 
440
+ # Build download dataframe with metadata for CSV
441
+ download_info_list = []
442
+ for nm in download_top["Name"]:
443
+ info = {}
444
+ meta = download_map.get(nm, {})
445
+ for col in meta_cols:
446
+ if col in meta and meta[col]:
447
+ info[col] = ", ".join(str(v) for v in meta[col] if pd.notna(v))
448
  else:
449
+ info[col] = ""
450
+ # attach totals
451
+ info["Total Value"] = int(download_top.loc[download_top["Name"] == nm, "Total Value"].iloc[0])
452
+ info["% of total"] = float(download_top.loc[download_top["Name"] == nm, "% of total"].iloc[0])
453
+ download_info_list.append(info)
 
454
 
 
 
 
 
 
455
  download_info_df = pd.DataFrame(download_info_list)
456
+ download_top = pd.concat([download_top.reset_index(drop=True), download_info_df.reset_index(drop=True)], axis=1)
457
 
458
+ return display_for_render, download_top
459
 
460
 
461
  def get_top_n_from_duckdb(
462
  con, group_col, top_n=10, time_filter=None, view="all_downloads"
463
  ):
464
  """
465
+ Query DuckDB directly to get model-level rows with per-model total_downloads (delta or full)
466
+ Returns rows similar to _get_filtered_top_n_from_duckdb in app.py.
 
 
 
 
 
 
 
 
467
  """
468
+ # Compute date window
469
+ if time_filter and len(time_filter) == 2:
 
470
  start = pd.to_datetime(time_filter[0], unit="s")
471
  end = pd.to_datetime(time_filter[1], unit="s")
472
+ else:
473
+ start = pd.to_datetime("1970-01-01")
474
+ # We cannot access end_dt here; rely on time_filter for end in typical use.
475
+ end = pd.Timestamp.now()
476
 
477
+ start_str = str(start)
478
+ end_str = str(end)
479
+
480
+ # If grouping by country, transform some country values
481
  if group_col == "org_country_single":
482
  group_expr = """CASE
483
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
 
487
  else:
488
  group_expr = group_col
489
 
490
+ # Derived author special-case
491
  if group_col == "derived_author":
492
  query = f"""
493
  WITH base_data AS (
494
+ SELECT
495
  {group_expr} AS group_key,
496
  CASE
497
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
498
+ WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
499
  ELSE org_country_single
500
  END AS org_country_single,
501
  author,
502
  derived_author,
503
  merged_country_groups_single,
504
  merged_modality,
505
+ model,
506
+ time,
507
+ downloadsAllTime
508
  FROM {view}
 
509
  ),
510
+
 
511
  author_country_lookup AS (
512
  SELECT DISTINCT
513
  author,
514
+ FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloadsAllTime DESC) AS author_country
515
  FROM base_data
516
  WHERE author IS NOT NULL
517
  ),
518
 
519
+ model_metrics AS (
520
+ SELECT
521
+ model,
522
+ group_key,
523
+ ANY_VALUE(org_country_single) AS org_country_single,
524
+ ANY_VALUE(author) AS author,
525
+ ANY_VALUE(derived_author) AS derived_author,
526
+ ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
527
+ ANY_VALUE(merged_modality) AS merged_modality,
528
+ COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
529
+ - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
530
+ AS total_downloads
531
  FROM base_data
532
+ GROUP BY model, group_key
533
  ),
534
 
535
+ total_downloads_cte AS (
536
+ SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
 
 
 
 
 
 
 
 
 
 
 
 
 
537
  )
538
 
539
+ SELECT
540
+ mm.model,
541
+ mm.group_key,
542
+ COALESCE(acl.author_country, mm.org_country_single) AS org_country_single,
543
+ mm.author,
544
+ mm.derived_author,
545
+ mm.merged_country_groups_single,
546
+ mm.merged_modality,
547
+ mm.total_downloads,
548
+ CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
549
+ FROM model_metrics mm
550
+ LEFT JOIN author_country_lookup acl ON mm.group_key = acl.author
551
+ CROSS JOIN total_downloads_cte td
552
+ WHERE mm.total_downloads > 0
553
+ ORDER BY mm.total_downloads DESC
554
+ LIMIT {top_n * 10};
555
  """
556
  else:
557
  query = f"""
558
  WITH base_data AS (
559
+ SELECT
560
  {group_expr} AS group_key,
561
  CASE
562
  WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
563
+ WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
564
  ELSE org_country_single
565
  END AS org_country_single,
566
  author,
567
  derived_author,
568
  merged_country_groups_single,
569
  merged_modality,
570
+ model,
571
+ time,
572
+ downloadsAllTime
573
  FROM {view}
 
574
  ),
575
 
576
+ model_metrics AS (
577
+ SELECT
578
+ model,
579
+ group_key,
580
+ ANY_VALUE(org_country_single) AS org_country_single,
581
+ ANY_VALUE(author) AS author,
582
+ ANY_VALUE(derived_author) AS derived_author,
583
+ ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
584
+ ANY_VALUE(merged_modality) AS merged_modality,
585
+ COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
586
+ - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
587
+ AS total_downloads
588
  FROM base_data
589
+ GROUP BY model, group_key
590
  ),
591
 
592
+ total_downloads_cte AS (
593
+ SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
 
 
 
 
 
 
 
 
 
 
 
 
594
  )
595
 
596
+ SELECT
597
+ mm.model,
598
+ mm.group_key,
599
+ mm.org_country_single,
600
+ mm.author,
601
+ mm.derived_author,
602
+ mm.merged_country_groups_single,
603
+ mm.merged_modality,
604
+ mm.total_downloads,
605
+ CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
606
+ FROM model_metrics mm
607
+ CROSS JOIN total_downloads_cte td
608
+ WHERE mm.total_downloads > 0
609
+ ORDER BY mm.total_downloads DESC
610
+ LIMIT {top_n * 10};
611
  """
612
 
613
  try: