Add aggregate scores and fix last column visibility

app.py

@@ -22,6 +22,7 @@ from src.display.utils import (
     EVAL_TYPES,
     NUMERIC_INTERVALS,
     TYPES,
+    TYPES_LITE,
     AutoEvalColumn,
     ModelType,
     fields,
@@ -83,9 +84,12 @@ def update_table(
     columns: list,
     query: str,
 ):
+    columns += "_" # The dataframe does not display the last column - BUG in gradio?
     #filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
     filtered_df = filter_queries(query, hidden_df)
     df = select_columns(filtered_df, columns)
+    print("TF")
+    print(df)
     return df
 
 
@@ -230,23 +234,23 @@ with demo:
                     )
                 """
             gr.Markdown(TABLE_DESC, elem_classes="markdown-text")
-            leaderboard_table = gr.components.Dataframe(
+            #print(shown_columns.value)
+            leaderboard_table = gr.Dataframe(
                 value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                    + shown_columns.value
-
+                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value
                 ],
                 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
+                datatype=TYPES_LITE,
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
-                wrap=False,
-                #column_widths=["2%", "2%"],
+                wrap=False
             )
+            print(leaderboard_table.value)
+            print(leaderboard_table.headers)
 
             # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+            hidden_leaderboard_table_for_search = gr.Dataframe(
                 value=original_df[COLS],
                 headers=COLS,
                 datatype=TYPES,
@@ -387,14 +391,14 @@ with demo:
                 elem_id="citation-button",
                 show_copy_button=True,
             )
-
+    
     demo.load(
         fn=generate_captcha,
         outputs=[captcha_img, text]
     )
 
-#scheduler = BackgroundScheduler()
-#scheduler.add_job(restart_space, "interval", seconds=3600)
-#scheduler.start()
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=86400)
+scheduler.start()
 demo.queue(default_concurrency_limit=40).launch(server_name="0.0.0.0")
 

src/display/about.py

@@ -88,6 +88,13 @@ All currently supported benchmarks are listed in the table below:
 | [Subjectivity](https://github.com/jirkoada/czechbench_eval_harness/tree/main/lm_eval/tasks/czechbench/subjectivity_cs)     | CS (Original)                 | Subjectivity Analysis      | Acc, Macro F1  | 2000    | subjectivity_cs |
 | [TruthfulQA](https://github.com/jirkoada/czechbench_eval_harness/tree/main/lm_eval/tasks/czechbench/truthfulqa_cs)         | CS (Translated)               | Knowledge-Based QA         | Acc            | 813     | truthfulqa_cs   |
 
+The leaderboard table also displays aggregated scores across task categories, including:
+- **Grammar (Avg.):** AGREE
+- **Knowledge (Avg.):** ARC-Challenge, ARC-Easy, MMLU, TruthfulQA
+- **Reasoning (Avg.):** ANLI, Belebele, CTKFacts, SQAD
+- **Math (Avg.):** GSM8K, Klokanek
+- **Classification (Avg.):** Czech News, Facebook Comments, Mall Reviews, Subjectivity
+
 ## Evaluation Process
 
 ### 1. Install CzechBench:

src/display/utils.py

@@ -51,6 +51,11 @@ auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_
 auto_eval_column_dict.append(["eval_name", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", True)])
 auto_eval_column_dict.append(["hf_model_id", ColumnContent, ColumnContent("Model URL", "str", False)])
+auto_eval_column_dict.append(["grammar_avg", ColumnContent, ColumnContent("Grammar (Avg.)", "number", True)])
+auto_eval_column_dict.append(["knowledge_avg", ColumnContent, ColumnContent("Knowledge (Avg.)", "number", True)])
+auto_eval_column_dict.append(["reasoning_avg", ColumnContent, ColumnContent("Reasoning (Avg.)", "number", True)])
+auto_eval_column_dict.append(["math_avg", ColumnContent, ColumnContent("Math (Avg.)", "number", True)])
+auto_eval_column_dict.append(["classification_avg", ColumnContent, ColumnContent("Classification (Avg.)", "number", True)])
 auto_eval_column_dict.append(["agree_cs", ColumnContent, ColumnContent("AGREE", "number", True)])
 auto_eval_column_dict.append(["anli_cs", ColumnContent, ColumnContent("ANLI", "number", True)])
 auto_eval_column_dict.append(["arc_challenge_cs", ColumnContent, ColumnContent("ARC-Challenge", "number", True)])
@@ -66,6 +71,8 @@ auto_eval_column_dict.append(["mmlu_cs", ColumnContent, ColumnContent("MMLU", "n
 auto_eval_column_dict.append(["sqad_cs", ColumnContent, ColumnContent("SQAD", "number", True)])
 auto_eval_column_dict.append(["subjectivity_cs", ColumnContent, ColumnContent("Subjectivity", "number", True)])
 auto_eval_column_dict.append(["truthfulqa_cs", ColumnContent, ColumnContent("TruthfulQA", "number", True)])
+auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("_", "str", True, dummy=True)])  # The dataframe does not display the last column - BUG in gradio?
+
 
 
 # We use make dataclass to dynamically fill the scores from Tasks
@@ -76,6 +83,7 @@ HEADER_MAP = {
     "eval_name": "Model",
     "precision": "Precision",
     "hf_model_id": "Model URL",
+    "knowledge_avg": "Knowledge (Avg.)",
     "agree_cs": "AGREE",
     "anli_cs": "ANLI",
     "arc_challenge_cs": "ARC-Challenge",
@@ -91,6 +99,7 @@ HEADER_MAP = {
     "sqad_cs": "SQAD",
     "subjectivity_cs": "Subjectivity",
     "truthfulqa_cs": "TruthfulQA",
+    "dummy": "_",
 }
 
 

src/populate.py

@@ -23,6 +23,12 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     #df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df.rename(columns=HEADER_MAP)
     df[df.select_dtypes(include=['number']).columns] *= 100 # convert to percentage
+    df["Grammar (Avg.)"] = df[["AGREE"]].mean(axis=1)
+    df["Knowledge (Avg.)"] = df[["ARC-Challenge", "ARC-Easy", "MMLU", "TruthfulQA"]].mean(axis=1)
+    df["Reasoning (Avg.)"] = df[["ANLI", "Belebele", "CTKFacts", "SQAD"]].mean(axis=1)
+    df["Math (Avg.)"] = df[["GSM8K", "Klokanek"]].mean(axis=1)
+    df["Classification (Avg.)"] = df[["Czech News", "Facebook Comments", "Mall Reviews", "Subjectivity"]].mean(axis=1)
+    df["_"] = ""  # The dataframe does not display the last column - BUG in gradio?
     df = df[cols].round(decimals=2)
     df.replace(r'\s+', np.nan, regex=True)
     # filter out if any of the benchmarks have not been produced