first version of czechbench leaderboard

app.py

@@ -1,3 +1,4 @@
+import json
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
@@ -53,48 +54,46 @@ except Exception:
 raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
 
+"""
 (
     finished_eval_queue_df,
     running_eval_queue_df,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
+"""
 
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame,
     columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
     query: str,
-):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    filtered_df = filter_queries(query, filtered_df)
+):  
+    #filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+    filtered_df = filter_queries(query, hidden_df)
     df = select_columns(filtered_df, columns)
     return df
 
 
 def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
+    print(query)
+    return df[(df[AutoEvalColumn.eval_name.name].str.contains(query, case=False))]
 
 
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
+        #AutoEvalColumn.model_type_symbol.name,
+        AutoEvalColumn.eval_name.name,
     ]
     # We use COLS to maintain sorting
     filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] #+ [AutoEvalColumn.dummy.name]
     ]
     return filtered_df
 
 
 def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
     final_df = []
-    if query != "":
+    if query != "" and query is not None:
         queries = [q.strip() for q in query.split(";")]
         for _q in queries:
             _q = _q.strip()
@@ -105,7 +104,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
         if len(final_df) > 0:
             filtered_df = pd.concat(final_df)
             filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
+                subset=[AutoEvalColumn.eval_name.name]
             )
 
     return filtered_df
@@ -115,23 +114,35 @@ def filter_models(
     df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
     # Show all models
-    if show_deleted:
-        filtered_df = df
-    else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
+    #if show_deleted:
+    #    filtered_df = df
+    #else:  # Show only still on the hub models
+    #    filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
+
+    filtered_df = df
 
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
+    #type_emoji = [t[0] for t in type_query]
+    #filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+    #filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
 
-    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
+    #numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
+    #params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+    #mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+    #filtered_df = filtered_df.loc[mask]
 
     return filtered_df
 
 
+def validate_upload(input):
+    try:
+        with open(input, mode="r") as f:
+            data = json.load(f)
+            #raise gr.Error("Cannot divide by zero!")
+    except:
+        raise gr.Error("Cannot parse file")
+    
+
+
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -163,10 +174,7 @@ with demo:
                             elem_id="column-select",
                             interactive=True,
                         )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
+                """    
                 with gr.Column(min_width=320):
                     # with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
@@ -190,19 +198,20 @@ with demo:
                         interactive=True,
                         elem_id="filter-columns-size",
                     )
-
+                """
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
                     [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                     + shown_columns.value
-                    + [AutoEvalColumn.dummy.name]
+                    
                 ],
                 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
-                column_widths=["2%", "33%"],
+                wrap=False,
+                #column_widths=["2%", "2%"],
             )
 
             # Dummy leaderboard for handling the case when the user uses backspace key
@@ -217,30 +226,18 @@ with demo:
                 [
                     hidden_leaderboard_table_for_search,
                     shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
                     search_bar,
                 ],
                 leaderboard_table,
             )
             for selector in [
                 shown_columns,
-                filter_columns_type,
-                filter_columns_precision,
-                filter_columns_size,
-                deleted_models_visibility,
             ]:
                 selector.change(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
                         shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        deleted_models_visibility,
                         search_bar,
                     ],
                     leaderboard_table,
@@ -254,8 +251,9 @@ with demo:
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
+                """
                 with gr.Column():
+                    
                     with gr.Accordion(
                         f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
                         open=False,
@@ -290,12 +288,18 @@ with demo:
                                 datatype=EVAL_TYPES,
                                 row_count=5,
                             )
+                    """
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
             with gr.Row():
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
+                
+                    file_output = gr.File()
+                    upload_button = gr.UploadButton("Upload json", file_types=['.json'])
+                    upload_button.upload(validate_upload, upload_button, file_output)
+                    """
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     model_type = gr.Dropdown(
                         choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
@@ -304,7 +308,8 @@ with demo:
                         value=None,
                         interactive=True,
                     )
-
+                    """
+                """
                 with gr.Column():
                     precision = gr.Dropdown(
                         choices=[i.value.name for i in Precision if i != Precision.Unknown],
@@ -321,18 +326,14 @@ with demo:
                         interactive=True,
                     )
                     base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
+                """
+            submit_button = gr.Button("Submit Eval", interactive=True)
             submission_result = gr.Markdown()
             submit_button.click(
                 add_new_eval,
                 [
                     model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
+                    upload_button
                 ],
                 submission_result,
             )
@@ -350,4 +351,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch(server_name="0.0.0.0")

src/display/about.py

@@ -12,9 +12,26 @@ class Task:
 # Init: to update with your specific keys
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    task0 = Task("task_agree", "accuracy", "AGREE")
-    task1 = Task("task_anli", "accuracy", "ANLI")
-
+    task0 = Task("task_agree", "accuracy", "agree")
+    task1 = Task("task_anli", "accuracy", "anli")
+    task2 = Task("task_agree_en", "accuracy", "anli_en")
+    task3 = Task("arc_challenge", "accuracy", "arc_challenge")
+    task4 = Task("arc_easy", "accuracy", "arc_easy")
+    task5 = Task("belebele", "accuracy", "belebele")
+    task6 = Task("ctkfacts", "accuracy", "ctkfacts")
+    task7 = Task("ctkfacts_en", "accuracy", "ctkfacts_en")
+    task8 = Task("czechnews", "accuracy", "czechnews")
+    task9 = Task("facebook_comments", "accuracy", "facebook_comments")
+    task10 = Task("klokánek", "accuracy", "klokánek")
+    task11 = Task("mall_reviews", "accuracy", "mall_reviews")
+    task12 = Task("mmlu", "accuracy", "mmlu")
+    task13 = Task("snli", "accuracy", "snli")
+    task14 = Task("snli_en", "accuracy", "snli_en")
+    task15 = Task("subjectivity", "accuracy", "subjectivity")
+    task16 = Task("subjectivity_en", "accuracy", "subjectivity_en")
+    task17 = Task("truthfulqa", "accuracy", "truthfulqa")
+    task18 = Task("gsm8k", "accuracy", "gsm8k")
+    task19 = Task("squad", "accuracy", "squad")
 
 # Your leaderboard name
 TITLE = """<h1 align="center" id="space-title">CzechBench example leaderboard</h1>"""

src/display/utils.py

@@ -25,6 +25,7 @@ class ColumnContent:
 
 ## Leaderboard columns
 auto_eval_column_dict = []
+"""
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
@@ -44,6 +45,29 @@ auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Avai
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 # Dummy column for the search bar (hidden by the custom CSS)
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+"""
+
+auto_eval_column_dict.append(["eval_name", ColumnContent, ColumnContent("eval_name", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["agree", ColumnContent, ColumnContent("agree", "number", True)])
+auto_eval_column_dict.append(["anli", ColumnContent, ColumnContent("anli", "number", True)])
+auto_eval_column_dict.append(["anli_en", ColumnContent, ColumnContent("anli_en", "number", True)])
+auto_eval_column_dict.append(["arc_challenge", ColumnContent, ColumnContent("arc_challenge", "number", True)])
+auto_eval_column_dict.append(["arc_easy", ColumnContent, ColumnContent("arc_easy", "number", True)])
+auto_eval_column_dict.append(["belebele", ColumnContent, ColumnContent("belebele", "number", True)])
+auto_eval_column_dict.append(["ctkfacts", ColumnContent, ColumnContent("ctkfacts", "number", True)])
+auto_eval_column_dict.append(["ctkfacts_en", ColumnContent, ColumnContent("ctkfacts_en", "number", True)])
+auto_eval_column_dict.append(["czechnews", ColumnContent, ColumnContent("czechnews", "number", True)])
+auto_eval_column_dict.append(["facebook_comments", ColumnContent, ColumnContent("facebook_comments", "number", True)])
+auto_eval_column_dict.append(["klokánek", ColumnContent, ColumnContent("klokánek", "number", True)])
+auto_eval_column_dict.append(["mall_reviews", ColumnContent, ColumnContent("mall_reviews", "number", True)])
+auto_eval_column_dict.append(["mmlu", ColumnContent, ColumnContent("mmlu", "number", True)])
+auto_eval_column_dict.append(["snli", ColumnContent, ColumnContent("snli", "number", True)])
+auto_eval_column_dict.append(["snli_en", ColumnContent, ColumnContent("snli_en", "number", True)])
+auto_eval_column_dict.append(["subjectivity", ColumnContent, ColumnContent("subjectivity", "number", True)])
+auto_eval_column_dict.append(["subjectivity_en", ColumnContent, ColumnContent("subjectivity_en", "number", True)])
+auto_eval_column_dict.append(["truthfulqa", ColumnContent, ColumnContent("truthfulqa", "number", True)])
+auto_eval_column_dict.append(["gsm8k", ColumnContent, ColumnContent("gsm8k", "number", True)])
+auto_eval_column_dict.append(["squad", ColumnContent, ColumnContent("squad", "number", True)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)

src/envs.py

@@ -4,13 +4,13 @@ from huggingface_hub import HfApi
 
 # clone / pull the lmeh eval data
 TOKEN = os.environ.get("TOKEN", None)
-
+print(TOKEN)
 OWNER = "davidadamczyk"
 REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+QUEUE_REPO = f"{OWNER}/leaderboard"
+RESULTS_REPO = f"{OWNER}/leaderboard"
 
-CACHE_PATH = os.getenv("HF_HOME", ".")
+CACHE_PATH = "." #os.getenv("HF_HOME", ".")
 
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")

src/leaderboard/read_evals.py

@@ -35,7 +35,7 @@ class EvalResult:
         """Inits the result from the specific model result file"""
         with open(json_filepath) as fp:
             data = json.load(fp)
-
+        return data
         config = data.get("config")
 
         # Precision
@@ -168,25 +168,10 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
-    eval_results = {}
+    eval_results = []
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict()  # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results
+        eval_results.append(eval_result)
+    
+    return eval_results

src/populate.py

@@ -10,10 +10,10 @@ from src.leaderboard.read_evals import get_raw_eval_results
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    #all_data_json = [v.to_dict() for v in raw_data]
+    df = pd.DataFrame.from_records(raw_data)
+    #df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced

src/submission/submit.py

@@ -1,9 +1,9 @@
 import json
 import os
 from datetime import datetime, timezone
-
+import numpy as np
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO, RESULTS_REPO
 from src.submission.check_validity import (
     already_submitted_models,
     check_model_card,
@@ -11,18 +11,61 @@ from src.submission.check_validity import (
     is_model_on_hub,
 )
 
+from src.display.utils import (
+    BENCHMARK_COLS,
+    COLS
+)
+
 REQUESTED_MODELS = None
 USERS_TO_SUBMISSION_DATES = None
 
 
 def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
+    eval_name: str,
+    upload: object
 ):
+    with open(upload, mode="r") as f:
+        data = json.load(f)    
+    
+    results = data['results']
+
+    ret = {"eval_name": eval_name}
+    # TODO add complex validation
+    print(results.keys())
+    print(BENCHMARK_COLS)
+    for input_col in results.keys():
+        if input_col not in BENCHMARK_COLS:
+            print(input_col)
+            return styled_error(f'Missing: {input_col}')
+    #ret.update({i:j['acc,none'] for i,j in results.items()})
+    ret.update({i:round(np.random.normal(1, 0.5, 1)[0], 2) for i,j in results.items()})
+    
+
+    user_name = "davidadamczyk"
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+
+    out_path = f"{OUT_DIR}/{eval_name}_eval_request.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(ret))
+
+
+    print("Uploading eval file")
+
+    print("path_or_fileobj: ", out_path)
+    print("path_in_repo: ",out_path.split("eval-queue/")[1])
+    print("repo_id: ", RESULTS_REPO)
+    print("repo_type: ", "dataset")
+
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=RESULTS_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {eval_name} to eval queue",
+    )
+
+    """
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
@@ -115,7 +158,7 @@ def add_new_eval(
 
     # Remove the local file
     os.remove(out_path)
-
+    """
     return styled_message(
         "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
     )