now with widget support

app.py

@@ -1,11 +1,15 @@
 import os
 os.system("wget https://raw.githubusercontent.com/Weyaxi/scrape-open-llm-leaderboard/main/openllm.py")
 from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, login
-from huggingface_hub.utils import RepositoryNotFoundError as deneme
-from openllm import *
-import gradio as gr
+from huggingface_hub import ModelCardData, EvalResult, ModelCard
+from huggingface_hub.repocard_data import eval_results_to_model_index
+from huggingface_hub.repocard import RepoCard
+from openllm import get_json_format_data, get_datas
+from tqdm import tqdm
+import time
 import requests
 import pandas as pd
+from pytablewriter import MarkdownTableWriter
 
 api = HfApi()
 fs = HfFileSystem()
@@ -25,49 +29,163 @@ def get_details_url(repo):
    return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}"
 
 
+def get_query_url(repo):
+  return f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"
+
+
+desc = """
+This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr
+
+The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.
+
+If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions
+"""
+
+
+def get_task_summary(results):
+  return {
+      "ARC":
+          {"dataset_type":"ai2_arc",
+          "dataset_name":"AI2 Reasoning Challenge (25-Shot)",
+          "dataset_short_name": "ARC (25-shot)",
+          "metric_type":"acc_norm",
+          "metric_value":results["ARC"],
+          "dataset_config":"ARC-Challenge",
+          "dataset_split":"test",
+          "dataset_revision":None,
+          "dataset_args":{"num_few_shot": 25},
+          "metric_name":"normalized accuracy"
+          },
+      "HellaSwag":
+          {"dataset_type":"hellaswag",
+          "dataset_name":"HellaSwag (10-Shot)",
+          "dataset_short_name": "HellaSwag (10-shot)",
+          "metric_type":"acc_norm",
+          "metric_value":results["HellaSwag"],
+          "dataset_config":None,
+          "dataset_split":"validation",
+          "dataset_revision":None,
+          "dataset_args":{"num_few_shot": 10},
+          "metric_name":"normalized accuracy"
+          },
+      "MMLU":
+      {
+          "dataset_type":"cais/mmlu",
+          "dataset_name":"MMLU (5-Shot)",
+          "dataset_short_name": "MMLU (5-Shot)",
+          "metric_type":"acc",
+          "metric_value":results["MMLU"],
+          "dataset_config":"all",
+          "dataset_split":"test",
+          "dataset_revision":None,
+          "dataset_args":{"num_few_shot": 5},
+          "metric_name":"accuracy"
+      },
+      "TruthfulQA":
+      {
+          "dataset_type":"truthful_qa",
+          "dataset_name":"TruthfulQA (0-shot)",
+          "dataset_short_name": "TruthfulQA (0-shot)",
+          "metric_type":"mc2",
+          "metric_value":results["TruthfulQA"],
+          "dataset_config":"multiple_choice",
+          "dataset_split":"validation",
+          "dataset_revision":None,
+          "dataset_args":{"num_few_shot": 0},
+          "metric_name":None
+      },
+      "Winogrande":
+      {
+          "dataset_type":"winogrande",
+          "dataset_name":"Winogrande (5-shot)",
+          "dataset_short_name": "Winogrande (5-shot)",
+          "metric_type":"acc",
+          "metric_value":results["Winogrande"],
+          "dataset_config":"winogrande_xl",
+          "dataset_split":"validation",
+          "dataset_args":{"num_few_shot": 5},
+          "metric_name":"accuracy"
+      },
+      "GSM8K":
+      {
+          "dataset_type":"gsm8k",
+          "dataset_name":"GSM8k (5-shot)",
+          "dataset_short_name": "GSM8k (5-shot)",
+          "metric_type":"acc",
+          "metric_value":results["GSM8K"],
+          "dataset_config":"main",
+          "dataset_split":"test",
+          "dataset_args":{"num_few_shot": 5},
+          "metric_name":"accuracy"
+      }
+  }
+
+
+
 def get_eval_results(repo):
   results = search(df, repo)
+  task_summary = get_task_summary(results)
+  md_writer = MarkdownTableWriter()
+  md_writer.headers = ["Metric", "Value"]
+  md_writer.value_matrix = [["Avg.", results['Average ⬆️']]] + [[v["dataset_short_name"], v["metric_value"]] for v in task_summary.items()]
 
   text = f"""
 # [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
 Detailed results can be found [here]({get_details_url(repo)})
 
-| Metric                | Value                     |
-|-----------------------|---------------------------|
-| Avg.                  | {results['Average ⬆️']}   |
-| ARC (25-shot)         | {results['ARC']}          |
-| HellaSwag (10-shot)   | {results['HellaSwag']}    |
-| MMLU (5-shot)         | {results['MMLU']}         |
-| TruthfulQA (0-shot)   | {results['TruthfulQA']}   |
-| Winogrande (5-shot)   | {results['Winogrande']}   |
-| GSM8K (5-shot)        | {results['GSM8K']}        |
+{md_writer.dumps()}
 """
   return text
 
 
-desc = """
-This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr
+def get_edited_yaml_readme(repo):
+  card = ModelCard.load(repo)
+  results = search(df, repo)
 
-The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.
+  common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open LLM Leaderboard", "source_url": f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"}
 
-If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions
-"""
+  tasks_results = get_task_summary(results)
+
+  if not card.data['eval_results']: # No results reported yet, we initialize the metadata
+    card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()])
+  else: # We add the new evaluations
+    for task in tasks_results.values():
+      cur_result = EvalResult(**task, **common)
+      if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']):
+        continue
+      card.data['eval_results'].append(cur_result)
 
+  return str(card)
+    
 
-def commit(hf_token, repo):
+def commit(hf_token, repo, pr_number=None, message="Adding Evaluation Results"): # specify pr number if you want to edit it, don't if you don't want
   login(hf_token)
+  edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True}
+
   try:
     try: # check if there is a readme already
-      readme_text = fs.read_text(f"{repo}/README.md") + get_eval_results(repo)
-    except:
-      readme_text = get_eval_results(repo)
+      readme_text = get_edited_yaml_readme(repo) + get_eval_results(repo)
+    except Exception as e:
+      if "Repo card metadata block was not found." in str(e): # There is no readme
+        readme_text = get_edited_yaml_readme(repo)
+      else:
+        print(f"Something went wrong: {e}")
 
     liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())]
-    commit = (create_commit(repo_id=repo, operations=liste, commit_message=f"Adding Evaluation Results", commit_description=desc, repo_type="model", create_pr=True).__dict__['pr_url'])
+    commit = (create_commit(repo_id=repo, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url)
+
     return commit
-  except Exception as e: # unexpected error
-    return e
 
+  except Exception as e:
+
+    if "Discussions are disabled for this repo" in str(e):
+      return "Discussions disabled"
+    elif "Cannot access gated repo" in str(e):
+      return "Gated repo"
+    elif "Repository Not Found" in str(e):
+      return "Repository Not Found"
+    else:
+      return e
 
 demo = gr.Interface(fn=commit, inputs=["text", "text"], outputs="text")