[ADD] Harness tasks, data display

.gitignore

@@ -10,4 +10,6 @@ eval-queue/
 eval-results/
 eval-queue-bk/
 eval-results-bk/
+eval-queue-local/
+eval-results-local/
 logs/

README.md

@@ -1,5 +1,5 @@
 ---
-title: MEDIC Benchmark
+title: Clinical NER Leaderboard
 emoji: 🥇
 colorFrom: green
 colorTo: indigo
@@ -7,8 +7,17 @@ sdk: gradio
 app_file: app.py
 pinned: true
 license: apache-2.0
+tags:
+  - leaderboard
+  - submission:automatic
+  - test:public
+  - judge:auto
+  - modality:text
 ---
 
+Also known as the NCER leaderboard HF, Huggingface. See the paper for more info: https://huggingface.co/papers/2410.05046.
+
+
 # Start the configuration
 
 Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
@@ -41,4 +50,4 @@ If you encounter problem on the space, don't hesitate to restart it to remove th
 You'll find 
 - the main table' columns names and properties in `src/display/utils.py`
 - the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -1,5 +1,6 @@
+import subprocess
+
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
@@ -9,30 +10,41 @@ from src.about import (
     CITATION_BUTTON_TEXT,
     EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
+    LLM_BENCHMARKS_TEXT_1,
+    EVALUATION_EXAMPLE_IMG,
+    LLM_BENCHMARKS_TEXT_2,
+    # ENTITY_DISTRIBUTION_IMG,
+    LLM_BENCHMARKS_TEXT_3,
     TITLE,
+    LOGO
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
+    DATASET_BENCHMARK_COLS,
+    TYPES_BENCHMARK_COLS,
+    DATASET_COLS,
+    Clinical_TYPES_COLS,
     EVAL_COLS,
     EVAL_TYPES,
+    NUMERIC_INTERVALS,
+    TYPES,
     AutoEvalColumn,
     ModelType,
-    fields,
+    ModelArch,
+    PromptTemplateName,
+    Precision,
     WeightType,
-    Precision
+    fields,
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+from src.submission.submit import add_new_eval, PLACEHOLDER_DATASET_WISE_NORMALIZATION_CONFIG
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-### Space initialisation
+
 try:
     print(EVAL_REQUESTS_PATH)
     snapshot_download(
@@ -48,8 +60,20 @@ try:
 except Exception:
     restart_space()
 
+# Span based results
+_, harness_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "accuracy", "datasets")
+harness_datasets_leaderboard_df = harness_datasets_original_df.copy()
+
+# _, span_based_types_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, Clinical_TYPES_COLS, TYPES_BENCHMARK_COLS, "SpanBasedWithPartialOverlap", "clinical_types")
+# span_based_types_leaderboard_df = span_based_types_original_df.copy()
+
+# # Token based results
+# _, token_based_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "datasets")
+# token_based_datasets_leaderboard_df = token_based_datasets_original_df.copy()
+
+# _, token_based_types_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, Clinical_TYPES_COLS, TYPES_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "clinical_types")
+# token_based_types_leaderboard_df = token_based_types_original_df.copy()
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 
 (
     finished_eval_queue_df,
@@ -57,51 +81,288 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
 
+def update_df(shown_columns, subset="datasets"):
+    leaderboard_table_df = harness_datasets_leaderboard_df.copy()
+    hidden_leader_board_df = harness_datasets_original_df
+    # else:
+    #     match evaluation_metric:
+    #         case "Span Based":
+    #             leaderboard_table_df = span_based_types_leaderboard_df.copy()
+    #             hidden_leader_board_df = span_based_types_original_df
+    #         case "Token Based":
+    #             leaderboard_table_df = token_based_types_leaderboard_df.copy()
+    #             hidden_leader_board_df = token_based_types_original_df
+    #         case _:
+    #             pass        
+    
+
+    value_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns
+
+    return leaderboard_table_df[value_cols], hidden_leader_board_df
+
+
+# Searching and filtering
+def update_table(
+    hidden_df: pd.DataFrame,
+    columns: list,
+    query: str,
+    type_query: list = None,
+    architecture_query: list = None,
+    size_query: list = None,
+    precision_query: str = None,
+    show_deleted: bool = False,
+):
+    filtered_df = filter_models(hidden_df, type_query, architecture_query, size_query, precision_query, show_deleted)
+    filtered_df = filter_queries(query, filtered_df)
+    df = select_columns(filtered_df, columns, list(hidden_df.columns))
+    return df
+
+
+def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
+
+
+def select_columns(df: pd.DataFrame, columns: list, cols:list) -> pd.DataFrame:
+    always_here_cols = [
+        AutoEvalColumn.model_type_symbol.name,
+        AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[always_here_cols + [c for c in cols if c in df.columns and c in columns]]
+    return filtered_df
+
+
+def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
+    final_df = []
+    if query != "":
+        queries = [q.strip() for q in query.split(";")]
+        for _q in queries:
+            _q = _q.strip()
+            if _q != "":
+                temp_filtered_df = search_table(filtered_df, _q)
+                if len(temp_filtered_df) > 0:
+                    final_df.append(temp_filtered_df)
+        if len(final_df) > 0:
+            filtered_df = pd.concat(final_df)
+            filtered_df = filtered_df.drop_duplicates(
+                subset=[
+                    AutoEvalColumn.model.name,
+                    # AutoEvalColumn.precision.name,
+                    # AutoEvalColumn.revision.name,
+                ]
+            )
+
+    return filtered_df
+
+
+def filter_models(
+    df: pd.DataFrame, type_query: list, architecture_query: list, size_query: list, precision_query: list, show_deleted: bool
+) -> pd.DataFrame:
+    # Show all models
+    # if show_deleted:
+    #     filtered_df = df
+    # else:  # Show only still on the hub models
+    #     filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
+
+    filtered_df = df
+
+    if type_query is not None:
+        type_emoji = [t[0] for t in type_query]
+        filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+
+    if architecture_query is not None:
+        arch_types = [t for t in architecture_query]
+        filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(arch_types)]
+            # filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(architecture_query + ["None"])]
+    
+    if precision_query is not None:
+        if AutoEvalColumn.precision.name in df.columns:
+            filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
+
+    if size_query is not None:
+        numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
+        params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+        mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+        filtered_df = filtered_df.loc[mask]
+
+    return filtered_df
+
+def change_submit_request_form(model_architecture):
+    match model_architecture:
+        case "Encoder":
+            return (
+                gr.Textbox(label="Threshold for gliner models", visible=False), 
+                gr.Radio(
+                        choices=["True", "False"],
+                        label="Load GLiNER Tokenizer",
+                        visible=False
+                    ),
+                gr.Dropdown(
+                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
+                        label="Prompt for generation",
+                        multiselect=False,
+                        # value="HTML Highlighted Spans",
+                        interactive=True,
+                        visible=False
+                    )
+            )
+        case "Decoder":
+            return (
+                gr.Textbox(label="Threshold for gliner models", visible=False), 
+                gr.Radio(
+                        choices=["True", "False"],
+                        label="Load GLiNER Tokenizer",
+                        visible=False
+                    ),
+                gr.Dropdown(
+                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
+                        label="Prompt for generation",
+                        multiselect=False,
+                        # value="HTML Highlighted Spans",
+                        interactive=True,
+                        visible=True
+                    )
+            )
+        case "GLiNER Encoder":
+            return (
+                gr.Textbox(label="Threshold for gliner models", visible=True), 
+                gr.Radio(
+                        choices=["True", "False"],
+                        label="Load GLiNER Tokenizer",
+                        visible=True
+                    ),
+                gr.Dropdown(
+                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
+                        label="Prompt for generation",
+                        multiselect=False,
+                        # value="HTML Highlighted Spans",
+                        interactive=True,
+                        visible=False
+                    )
+            )
 
+            
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
+    gr.HTML(LOGO, elem_classes="logo")
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+        with gr.TabItem("🏅 Closed Ended Evaluation", elem_id="llm-benchmark-tab-table", id=0):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and not c.clinical_type_col],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and not c.clinical_type_col
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                with gr.Column(min_width=320):
+                    # with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model Types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    # filter_columns_architecture = gr.CheckboxGroup(
+                    #     label="Architecture Types",
+                    #     choices=[i.value.name for i in ModelArch],
+                    #     value=[i.value.name for i in ModelArch],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-architecture",
+                    # )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="datasets")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            # hidden_leaderboard_table_for_search = gr.components.Dataframe(
+            #     value=datasets_original_df[DATASET_COLS],
+            #     headers=DATASET_COLS,
+            #     datatype=TYPES,
+            #     visible=False,
+            # )
+
+                        
+            # search_bar.submit(
+            #     update_table,
+            #     [
+            #         hidden_leaderboard_table_for_search,
+            #         shown_columns,
+            #         search_bar,
+            #         filter_columns_type,
+            #         # filter_columns_architecture
+            #     ],
+            #     leaderboard_table,
+            # )
+            # for selector in [
+            #     shown_columns,
+            #     filter_columns_type,
+            #     # filter_columns_architecture,
+            #     # filter_columns_size,
+            #     # deleted_models_visibility,
+            # ]:
+            #     selector.change(
+            #         update_table,
+            #         [
+            #             hidden_leaderboard_table_for_search,
+            #             shown_columns,
+            #             search_bar,
+            #             filter_columns_type,
+            #             # filter_columns_architecture,
+            #         ],
+            #         leaderboard_table,
+            #         queue=True,
+            #     )
+
+        with gr.TabItem("🏅 Open Ended Evaluation", elem_id="llm-benchmark-tab-table", id=1):
+            pass
+        with gr.TabItem("🏅 Med Safety", elem_id="llm-benchmark-tab-table", id=2):
+            pass
+
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):
+            gr.Markdown(LLM_BENCHMARKS_TEXT_1, elem_classes="markdown-text")
+            gr.HTML(EVALUATION_EXAMPLE_IMG, elem_classes="logo")
+            gr.Markdown(LLM_BENCHMARKS_TEXT_2, elem_classes="markdown-text")
+            # gr.HTML(ENTITY_DISTRIBUTION_IMG, elem_classes="logo")
+            gr.Markdown(LLM_BENCHMARKS_TEXT_3, elem_classes="markdown-text")
+
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=4):
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
@@ -146,8 +407,16 @@ with demo:
 
             with gr.Row():
                 with gr.Column():
+
                     model_name_textbox = gr.Textbox(label="Model name")
+
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+
+                    model_arch = gr.Radio(
+                        choices=[t.to_str(" : ") for t in ModelArch if t != ModelArch.Unknown],
+                        label="Model Architecture",
+                    )
+                    
                     model_type = gr.Dropdown(
                         choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
                         label="Model type",
@@ -157,21 +426,29 @@ with demo:
                     )
 
                 with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
+                    label_normalization_map = gr.Textbox(lines=6, label="Label Normalization Map", placeholder=PLACEHOLDER_DATASET_WISE_NORMALIZATION_CONFIG)
+                    gliner_threshold = gr.Textbox(label="Threshold for GLiNER models", visible=False)
+                    gliner_tokenizer_bool = gr.Radio(
+                        choices=["True", "False"],
+                        label="Load GLiNER Tokenizer",
+                        visible=False
                     )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
+                    prompt_name = gr.Dropdown(
+                        choices=[prompt_template.value for prompt_template in PromptTemplateName],
+                        label="Prompt for generation",
                         multiselect=False,
-                        value="Original",
+                        value="HTML Highlighted Spans",
                         interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                        visible=False
+                    )# should be a dropdown
+
+                    # parsing_function - this is tied to the prompt & therefore does not need to be specified
+                    # generation_parameters = gr.Textbox(label="Generation params in json format") just default for now
+
+                    model_arch.change(fn=change_submit_request_form, inputs=model_arch, outputs=[
+                        gliner_threshold, 
+                        gliner_tokenizer_bool, 
+                        prompt_name])
 
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
@@ -179,15 +456,20 @@ with demo:
                 add_new_eval,
                 [
                     model_name_textbox,
-                    base_model_name_textbox,
+                    # base_model_name_textbox,
                     revision_name_textbox,
-                    precision,
-                    weight_type,
+                    model_arch,
+                    label_normalization_map,
+                    gliner_threshold,
+                    gliner_tokenizer_bool,
+                    prompt_name,
+                    # weight_type,
                     model_type,
                 ],
                 submission_result,
             )
 
+
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(
@@ -201,4 +483,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch(allowed_paths=['./assets/'])

eval_metrics_app.py

@@ -0,0 +1,75 @@
+import gradio as gr
+
+# Function to compute evaluation metrics (dummy implementation)
+def compute_metrics(gt_spans, pred_spans):
+    # Dummy implementation of a metric computation
+    # Replace this with actual metric computation logic
+    tp = len(set(gt_spans) & set(pred_spans))
+    fp = len(set(pred_spans) - set(gt_spans))
+    fn = len(set(gt_spans) - set(pred_spans))
+    precision = tp / (tp + fp) if (tp + fp) > 0 else 0
+    recall = tp / (tp + fn) if (tp + fn) > 0 else 0
+    f1_score = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0
+    
+    return {"precision": precision, "recall": recall, "f1_score": f1_score}
+
+def create_app():
+    with gr.Blocks() as demo:
+        # Input components
+        text_input = gr.Textbox(label="Input Text")
+        highlight_input = gr.Textbox(label="Highlight Text and Press Add")
+        
+        gt_spans_state = gr.State([])
+        pred_spans_state = gr.State([])
+
+        # Buttons for ground truth and prediction
+        add_gt_button = gr.Button("Add to Ground Truth")
+        add_pred_button = gr.Button("Add to Predictions")
+
+        # Outputs for highlighted spans
+        gt_output = gr.HighlightedText(label="Ground Truth Spans")
+        pred_output = gr.HighlightedText(label="Predicted Spans")
+
+        # Compute metrics button and its output
+        compute_button = gr.Button("Compute Metrics")
+        metrics_output = gr.JSON(label="Metrics")
+
+        # Function to update spans
+        def update_spans(text, span, gt_spans, pred_spans, is_gt):
+            start_idx = text.find(span)
+            end_idx = start_idx + len(span)
+            new_span = (start_idx, end_idx)
+            if is_gt:
+                gt_spans.append(new_span)
+                gt_spans = list(set(gt_spans))
+            else:
+                pred_spans.append(new_span)
+                pred_spans = list(set(pred_spans))
+            return gt_spans, pred_spans, highlight_spans(text, gt_spans), highlight_spans(text, pred_spans)
+        
+        # Function to highlight spans
+        def highlight_spans(text, spans):
+            span_dict = {}
+            for span in spans:
+                span_dict[(span[0], span[1])] = "highlight"
+            return span_dict
+
+        # Event handlers for buttons
+        add_gt_button.click(fn=update_spans, inputs=[text_input, highlight_input, gt_spans_state, pred_spans_state, gr.State(True)], outputs=[gt_spans_state, pred_spans_state, gt_output, pred_output])
+        add_pred_button.click(fn=update_spans, inputs=[text_input, highlight_input, gt_spans_state, pred_spans_state, gr.State(False)], outputs=[gt_spans_state, pred_spans_state, gt_output, pred_output])
+
+        # Function to compute metrics
+        def on_compute_metrics(gt_spans, pred_spans):
+            metrics = compute_metrics(gt_spans, pred_spans)
+            return metrics
+        
+        compute_button.click(fn=on_compute_metrics, inputs=[gt_spans_state, pred_spans_state], outputs=metrics_output)
+
+        # Layout arrangement
+        text_input.change(fn=lambda x: x, inputs=text_input, outputs=[gt_output, pred_output])
+
+    return demo
+
+# Run the app
+demo = create_app()
+demo.launch()

medic-harness-requests/.gitattributes

@@ -0,0 +1,58 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

medic-harness-results/.gitattributes

@@ -0,0 +1,58 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

medic-harness-results/aaditya/Llama3-OpenBioLLM-70B/results_2024-07-24T15:26:36Z.json

@@ -0,0 +1,37 @@
+{
+  "config": {
+    "model_name": "aaditya/Llama3-OpenBioLLM-70B",
+    "revision": "main",
+    "submitted_time": "2024-07-24 14:33:56+00:00",
+    "model_type": "domain-specific",
+    "num_params": 70000000000,
+    "private": false,
+    "evaluated_time": "2024-07-24T15:26:36Z"
+  },
+  "results": {
+    "MMLU": {
+      "accuracy": 90.4
+    },
+    "MMLU-Pro": {
+      "accuracy": 64.2
+    },
+    "MedMCQA": {
+      "accuracy": 73.2
+    },
+    "MedQA": {
+      "accuracy": 76.9
+    },
+    "USMLE": {
+      "accuracy": 79.0
+    },
+    "PubMedQA": {
+      "accuracy": 73.2
+    },
+    "ToxiGen": {
+      "accuracy": 91.3
+    },
+    "Average": {
+      "accuracy": 78.3
+    }
+  }
+}

medic-harness-results/meta-llama/Llama-3.1-8B-Instruct/results_2024-07-24T15:26:36Z.json

@@ -0,0 +1,39 @@
+{
+  "config": {
+    "model_name": "meta-llama/Llama-3.1-8B-Instruct",
+    "revision": "main",
+    "submitted_time": "2024-07-24 14:33:56+00:00",
+    "model_type": "instruct-tuned",
+    "num_params": 8000000000,
+    "private": false,
+    "evaluated_time": "2024-07-24T15:26:36Z"
+  },
+  "results": {
+    "MMLU": {
+      "accuracy": 73.4
+    },
+    "MMLU-Pro": {
+      "accuracy": 49.9
+    },
+    "MedMCQA": {
+      "accuracy": 58.4
+    },
+    "MedQA": {
+      "accuracy": 62.0
+    },
+    "USMLE": {
+      "accuracy": 68.2
+    },
+    "PubMedQA": {
+      "accuracy": 76.2
+    },
+    "ToxiGen": {
+      "accuracy": 82.3
+    },
+    "Average": {
+      "accuracy": 67.2
+    }
+  }
+}
+
+

requirements.txt

@@ -1,16 +1,18 @@
 APScheduler
 black
+click
 datasets
 gradio
-gradio[oauth]
-gradio_leaderboard==0.0.9
 gradio_client
 huggingface-hub>=0.18.0
 matplotlib
 numpy
 pandas
 python-dateutil
+requests
 tqdm
 transformers
 tokenizers>=0.15.0
+git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
+accelerate
 sentencepiece

src/about.py

@@ -1,72 +1,267 @@
 from dataclasses import dataclass
 from enum import Enum
 
+
 @dataclass
-class Task:
+class HarnessTask:
     benchmark: str
     metric: str
     col_name: str
-
+  
 
 # Select your tasks here
 # ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+class HarnessTasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    # task0 = Task("anli_r1", "acc", "ANLI")
+    # task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task0 = HarnessTask("MMLU", "accuracy", "MMLU")
+    task1 = HarnessTask("MMLU-Pro", "accuracy", "MMLU-Pro")
+    task2 = HarnessTask("MedMCQA", "accuracy", "MedMCQA")
+    task3 = HarnessTask("MedQA", "accuracy", "MedQA")
+    task4 = HarnessTask("USMLE", "accuracy", "USMLE")
+    task5 = HarnessTask("PubMedQA", "accuracy", "PubMedQA")
+    task6 = HarnessTask("ToxiGen", "accuracy", "ToxiGen")
+    task7 = HarnessTask("Average", "accuracy", "Harness-Average")
+    # task5 = Task("", "f1", "")
+    # task6 = Task("", "f1", "")
+
+@dataclass
+class ClinicalType:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class ClinicalTypes(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    type0 = ClinicalType("condition", "f1", "CONDITION") 
+    type1 = ClinicalType("measurement", "f1", "MEASUREMENT") 
+    type2 = ClinicalType("drug", "f1", "DRUG") 
+    type3 = ClinicalType("procedure", "f1", "PROCEDURE") 
+    type4 = ClinicalType("gene", "f1", "GENE")
+    type5 = ClinicalType("gene variant", "f1", "GENE VARIANT")
 
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
 
+NUM_FEWSHOT = 0  # Change with your few shot
+# ---------------------------------------------------
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """""" #<h1 align="center" id="space-title"> NER Leaderboard</h1>"""
+LOGO = """<img src="https://equalengineers.com/wp-content/uploads/2024/04/dummy-logo-5b.png" alt="Clinical X HF" width="500" height="333">"""
+# LOGO = """<img src="https://huggingface.co/spaces/m42-health/clinical_ner_leaderboard/resolve/main/assets/image.png" alt="Clinical X HF" width="500" height="333">"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+The rapid development of Large Language Models (LLMs) for healthcare applications has spurred calls for holistic evaluation beyond frequently-cited benchmarks like USMLE, to better reflect real-world performance. While real-world assessments are valuable indicators of utility, they often lag behind the pace of LLM evolution, likely rendering findings obsolete upon deployment. This temporal disconnect necessitates a comprehensive upfront evaluation that can guide model selection for specific clinical applications. We introduce MEDIC, a framework assessing LLMs across five critical dimensions of clinical competence: medical reasoning, ethics and bias, data and language understanding, in-context learning, and clinical safety. MEDIC features a novel cross-examination framework quantifying LLM performance across areas like coverage and hallucination detection, without requiring reference outputs. We apply MEDIC to evaluate LLMs on medical question-answering, safety, summarization, note generation, and other tasks. Our results show performance disparities across model sizes, baseline vs medically finetuned models, and have implications on model selection for applications requiring specific model strengths, such as low hallucination or lower cost of inference. MEDIC's multifaceted evaluation reveals these performance trade-offs, bridging the gap between theoretical capabilities and practical implementation in healthcare settings, ensuring that the most promising models are identified and adapted for diverse healthcare applications.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
+LLM_BENCHMARKS_TEXT_1 = f"""
+
+## About
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+The Named Clinical Entity Recognition Leaderboard is aimed at advancing the field of natural language processing in healthcare. It provides a standardized platform for evaluating and comparing the performance of various language models in recognizing named clinical entities, a critical task for applications such as clinical documentation, decision support, and information extraction. By fostering transparency and facilitating benchmarking, the leaderboard's goal is to drive innovation and improvement in NLP models. It also helps researchers identify the strengths and weaknesses of different approaches, ultimately contributing to the development of more accurate and reliable tools for clinical use. Despite its exploratory nature, the leaderboard aims to play a role in guiding research and ensuring that advancements are grounded in rigorous and comprehensive evaluations. 
 
+## Evaluation method and metrics
+When training a Named Entity Recognition (NER) system, the most common evaluation methods involve measuring precision, recall, and F1-score at the token level. While these metrics are useful for fine-tuning the NER system, evaluating the predicted named entities for downstream tasks requires metrics at the full named-entity level. We include both evaluation methods: token-based and span-based. We provide an example below which helps in understanding the difference between the methods.
+Example Sentence: "The patient was diagnosed with a skin cancer disease."
+For simplicity, let's assume the an example sentence which contains 10 tokens, with a single two-token disease entity (as shown in the figure below).
 """
+EVALUATION_EXAMPLE_IMG = """<img src="https://huggingface.co/spaces/m42-health/clinical_ner_leaderboard/resolve/main/assets/ner_evaluation_example.png" alt="Clinical X HF" width="750" height="500">"""
+LLM_BENCHMARKS_TEXT_2 = """
+Token-based evaluation involves obtaining the set of token labels (ground-truth annotations) for the annotated entities and the set of token predictions, comparing these sets, and computing a classification report. Hence, the results for the example above are shown below.
+**Token-based metrics:**
 
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+  
+  
+| Model   | TP  | FP  | FN  | Precision | Recall | F1-Score |
+| ------- | --- | --- | --- | --------- | ------ | -------- |
+| Model D | 0   | 1   | 0   | 0.00      | 0.00   | 0.00     |
+| Model C | 1   | 1   | 1   | 0.50      | 0.50   | 0.50     |
+| Model B | 2   | 2   | 0   | 0.50      | 1.00   | 0.67     |
+| Model A | 2   | 1   | 0   | 0.67      | 1.00   | 0.80     |
+
+
+Where,
+$$ Precision = TP / (TP + FP)$$
+$$ Recall = TP / (TP + FN)$$
+$$ f1score = 2 * (Prec * Rec) / (Prec + Rec)$$
+  
+  
+
+With this token-based approach, we have a broad idea of the performance of the model at the token level. However, it may misrepresent the performance at the entity level when the entity includes more than 1 token (which may be more relevant for certain applications). In addition, depending on the annotations of certain datasets, we may not want to penalize a model for a "partial" match with a certain entity.  
+The span-based method attempts to address some of these issues, by determining the full or partial matches at the entity level to classify the predictions as correct, incorrect, missed and spurious. These are then used to calculate precision, recall and F1-score. Given this, for the example below.  
+ 
+**Span-based metrics:**
+
+
+| Model   | Correct | Incorrect | Missed | Spurious | Precision | Recall | F1-Score |
+| ------- | ------- | --------- | ------ | -------- | --------- | ------ | -------- |
+| Model A | 1       | 0         | 0      | 0        | 1.00      | 1.00   | 1.00     |
+| Model B | 1       | 0         | 0      | 0        | 1.00      | 1.00   | 1.00     |
+| Model C | 1       | 0         | 0      | 0        | 1.00      | 1.00   | 1.00     |
+| Model D | 0       | 0         | 1      | 1        | 0.00      | 0.00   | 0.00     |
+
+
+Where,
+$$ Precision = COR / (COR + INC + SPU)$$
+$$ Recall = COR / (COR + INC + MIS)$$
+$$ f1score = 2 * (Prec * Rec) / (Prec + Rec)$$
+
+Note:
+1. Span-based approach here is equivalent to the 'Span Based Evaluation with Partial Overlap' in [NER Metrics Showdown!](https://huggingface.co/spaces/wadood/ner_evaluation_metrics) and is equivalent to Partial Match ("Type") in the nervaluate python package.
+2. Token-based approach here is equivalent to the 'Token Based Evaluation With Macro Average' in [NER Metrics Showdown!](https://huggingface.co/spaces/wadood/ner_evaluation_metrics)
+
+Additional examples can be tested on the [NER Metrics Showdown!](https://huggingface.co/spaces/wadood/ner_evaluation_metrics) huggingface space.
+
+## Datasets
+The following datasets (test splits only) have been included in the evaluation. 
+
+### [NCBI Disease](https://huggingface.co/datasets/m42-health/clinical_ncbi)
+The NCBI Disease corpus includes mention and concept level annotations on PubMed abstracts. It covers annotations of diseases. 
+
+|            | Counts |
+| ---------- | ------ |
+| Samples    | 100    |
+| Annotation | 960    |
+
+
+### [CHIA](https://huggingface.co/datasets/m42-health/clinical_chia)
+This is a large, annotated corpus of patient eligibility criteria extracted from registered clinical trials (ClinicalTrials.gov). Annotations cover 15 different entity types, including conditions, drugs, procedures, and measurements.
+
+
+|            | Counts |
+| ---------- | ------ |
+| Samples    | 194    |
+| Annotation | 3981   |
+
+
+### [BC5CDR](https://huggingface.co/datasets/m42-health/clinical_bc5cdr)
+The BC5CDR corpus consists of 1500 PubMed articles with annotated chemicals and diseases.
+
+
+|            | Counts |
+| ---------- | ------ |
+| Samples    | 500    |
+| Annotation | 9928   |
+
+
+### [BIORED](https://huggingface.co/datasets/m42-health/clinical_biored)
+The BIORED corpus includes a set of PubMed abstracts with annotations of multiple entity types (e.g., gene/protein, disease, chemical).
+
+
+|            | Counts |
+| ---------- | ------ |
+| Samples    | 100    |
+| Annotation | 3535   |
+
+
+Datasets summary
+
+A summary of the datasets used are summarized here.
+
+
+| Dataset | # samples | # annotations | # original entities | # clinical entities |
+| ------- | --------- | ------------- | ------------------- | ------------------- |
+| NCBI    | 100       | 960           | 4                   | 1                   |
+| CHIA    | 194       | 3981          | 16                  | 4                   |
+| BIORED  | 500       | 9928          | 2                   | 4                   |
+| BC5CDR  | 100       | 3535          | 6                   | 2                   |
+
+
+## Clinical Entity Types
+
+The above datasets are modified to cater to the clinical setting. For this, the entity types that are clinically relevant are retained and the rest are dropped. Further, the clinical entity type is standardized across the dataset to obtain a total of 6 clinical entity types shown below.
+
+
+| Clinical Entity | Combined Annotation |
+| --------------- | ------------------- |
+| Condition       | 7514                |
+| Drug            | 6443                |
+| Procedure       | 300                 |
+| Measurement     | 258                 |
+| Gene            | 1180                |
+| Gene Variant    | 241                 |
+
+
+"""
+
+ENTITY_DISTRIBUTION_IMG = """<img src="file/assets/entity_distribution.png" alt="Clinical X HF" width="750" height="500">"""
+LLM_BENCHMARKS_TEXT_3="""
+## Decoder Model Evaluation
+Evaluating encoder models, such as BERT, for token classification tasks (e.g., NER) is straightforward given that these models process the entire input sequence simultaneously. This allows them to generate token-level classifications by leveraging bidirectional context, facilitating a direct comparison of predicted tags against the gold standard labels for each token in the input sequence.  
+
+In contrast, decoder-only models, like GPT, generate responses sequentially, predicting one token at a time based on the preceding context. Evaluating the performance of these models for token classification tasks requires a different approach. First, we prompt the decoder-only LLM with a specific task of tagging the different entity types within a given text. This task is clearly defined to the model, ensuring it understands which types of entities to identify (i.e., conditions, drugs, procedures, etc).   
+An example of the task prompt is shown below.
+```
+## Instruction
+Your task is to generate an HTML version of an input text, marking up specific entities related to healthcare. The entities to be identified are: symptom, disorder. Use HTML <span > tags to highlight these entities. Each <span > should have a class attribute indicating the type of the entity. Do NOT provide further examples and just consider the input provided below. Do NOT provide an explanation nor notes about the reasoning. Do NOT reformat nor summarize the input text. Follow the instruction and the format of the example below.
+ 
+## Entity markup guide
+Use <span class='symptom' > to denote a symptom.
+Use <span class='disorder' > to denote a disorder.
 ```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+To ensure deterministic and consistent outputs, the temperature for generation is kept at 0.0. The model then generates a sequential response that includes the tagged entities, as shown in the example below.
+```
+## Input:
+He had been diagnosed with osteoarthritis of the knees and had undergone arthroscopy years prior to admission.
+## Output:
+He had been diagnosed with <span class="disease" >osteoarthritis of the knees</span >and had undergone <span class="procedure" >arthroscopy</span >years prior to admission.
+```
+
+After the tagged output is generated, it is parsed to extract the tagged entities. The parsed data are then compared against the gold standard labels, and performance metrics are computed as above. This evaluation method ensures a consistent and objective assessment of decoder-only LLM's performance in NER tasks, despite the differences in their architecture compared to encoder models. 
+
+# Reproducibility
+To reproduce our results, follow the steps detailed [here](https://github.com/WadoodAbdul/clinical_ner_benchmark/blob/master/docs/reproducing_results.md)
+
+# Disclaimer and Advisory
+The Leaderboard is maintained by the authors and affiliated entity as part of our ongoing contribution to open research in the field of NLP in healthcare. The leaderboard is intended for academic and exploratory purposes only. The language models evaluated on this platform (to the best knowledge of the authors) have not been approved for clinical use, and their performance should not be interpreted as clinically validated or suitable for real-world medical applications.  
+
+Users are advised to approach the results with an understanding of the inherent limitations and the experimental nature of this evaluation. The authors and affiliated entity do not endorse any specific model or approach, and the leaderboard is provided without any warranties or guarantees. Researchers and practitioners are encouraged to use the leaderboard as a resource to guide further research and development, keeping in mind the necessity for rigorous testing and validation in clinical settings before any practical application.
+
+
+
+
+"""
+
+EVALUATION_QUEUE_TEXT = """
+
+Currently, the benchmark supports evaluation for models hosted on the huggingface hub and of type encoder, decoder or gliner type models.
+If your model needs a custom implementation, follow the steps outlined in the [clinical_ner_benchmark](https://github.com/WadoodAbdul/clinical_ner_benchmark/blob/e66eb566f34e33c4b6c3e5258ac85aba42ec7894/docs/custom_model_implementation.md) repo or reach out to our team!
+
+
+### Fields Explanation
+
+#### Model Type:
+- Fine-Tuned: If the training data consisted of any split/variation of the datasets on the leaderboard.  
+- Zero-Shot: If the model did not have any exposure to the datasets on the leaderboard while training.
 
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+#### Model Architecture:
+- Encoder: The standard transformer encoder architecture with a token classification head on top.  
+- Decoder: Transformer based autoregressive token generation model.  
+- GLiNER: Architecture outlined in the [GLiNER Paper](https://arxiv.org/abs/2311.08526)
 
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+#### Label Normalization Map:
+Not all models have been tuned to output the ner label names in the clinical datasets on this leaderboard. Some models cater to the same entity names with a synonym of it.
+The normalization map can be used to ensure that the models's output are aligned with the labels expected in the datasets.
 
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+Note: Multiple model labels can be mapped to a single entity type in the leaderboard dataset. Ex: 'synonym' and 'disease' to 'condition'
 
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
 
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+Upon successful submission of your request, your model's result would be updated on the leaderboard within 5 working days!
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
+@misc{abdul2024namedclinicalentityrecognition,
+      title={Named Clinical Entity Recognition Benchmark}, 
+      author={Wadood M Abdul and Marco AF Pimentel and Muhammad Umar Salman and Tathagata Raha and Clément Christophe and Praveen K Kanithi and Nasir Hayat and Ronnie Rajan and Shadab Khan},
+      year={2024},
+      eprint={2410.05046},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2410.05046}, 
+}
+ 
 """

src/display/css_html_js.py

@@ -1,4 +1,11 @@
 custom_css = """
+.logo {
+    width: 500px;
+    height: auto;
+    margin: 0 auto;
+    max-width: 100%
+    object-fit: contain;
+}
 
 .markdown-text {
     font-size: 16px !important;

src/display/utils.py

@@ -3,7 +3,9 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import HarnessTasks
+from src.about import ClinicalTypes
+
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -19,53 +21,64 @@ class ColumnContent:
     displayed_by_default: bool
     hidden: bool = False
     never_hidden: bool = False
+    dataset_task_col: bool = False
+    clinical_type_col: bool = False
+
 
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True)])
+for task in HarnessTasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, False, dataset_task_col=True)])
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["backbone", ColumnContent, ColumnContent("Base Model", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False, True)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False, True)])
+auto_eval_column_dict.append(
+    ["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, True)]
+)
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, True)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
+    architecture = ColumnContent("model_architecture", "bool", True)
+    # precision = ColumnContent("precision", "str", True)
+    # weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
     display_name: str = ""
-    symbol: str = "" # emoji
+    symbol: str = ""  # emoji
 
 
 class ModelType(Enum):
+    ZEROSHOT = ModelDetails(name="zero-shot", symbol="⚫")
+    FINETUNED = ModelDetails(name="fine-tuned", symbol="⚪")
     PT = ModelDetails(name="pretrained", symbol="🟢")
     FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    # IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    # RL = ModelDetails(name="RL-tuned", symbol="🟦")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -73,24 +86,55 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
+        if "zero-shot" in type or "⚫" in type:
+            return ModelType.ZEROSHOT
+        if "fine-tuned" in type or "⚪" in type:
+            return ModelType.FINETUNED
+        # if "fine-tuned" in type or "🔶" in type:
+        #     return ModelType.FT
+        # if "pretrained" in type or "🟢" in type:
+        #     return ModelType.PT
+        # if "RL-tuned" in type or "🟦" in type:
+        #     return ModelType.RL
+        # if "instruction-tuned" in type or "⭕" in type:
+        #     return ModelType.IFT
         return ModelType.Unknown
 
+class ModelArch(Enum):
+    Encoder = ModelDetails("Encoder")
+    Decoder = ModelDetails("Decoder")
+    GLiNEREncoder = ModelDetails("GLiNER Encoder")
+    Unknown = ModelDetails(name="Other", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "Encoder" == type:
+            return ModelArch.Encoder
+        if "Decoder" == type:
+            return ModelArch.Decoder
+        if "GLiNER Encoder" == type:
+            return ModelArch.GLiNEREncoder
+        # if "unknown" in type:
+        #     return ModelArch.Unknown
+        return ModelArch.Unknown
+
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
+    float32 = ModelDetails("float32")
+    # qt_8bit = ModelDetails("8bit")
+    # qt_4bit = ModelDetails("4bit")
+    # qt_GPTQ = ModelDetails("GPTQ")
     Unknown = ModelDetails("?")
 
     def from_str(precision):
@@ -98,13 +142,49 @@ class Precision(Enum):
             return Precision.float16
         if precision in ["torch.bfloat16", "bfloat16"]:
             return Precision.bfloat16
+        if precision in ["float32"]:
+            return Precision.float32
+        # if precision in ["8bit"]:
+        #    return Precision.qt_8bit
+        # if precision in ["4bit"]:
+        #    return Precision.qt_4bit
+        # if precision in ["GPTQ", "None"]:
+        #    return Precision.qt_GPTQ
         return Precision.Unknown
 
+
+class PromptTemplateName(Enum):
+    UniversalNERTemplate = "universal_ner"
+    LLMHTMLHighlightedSpansTemplate = "llm_html_highlighted_spans"
+    LLMHTMLHighlightedSpansTemplateV1 = "llm_html_highlighted_spans_v1"
+    LLamaNERTemplate = "llama_70B_ner"
+    # MixtralNERTemplate = "mixtral_ner_v0.3"
+
+class EvaluationMetrics(Enum):
+    SpanBased = "Span Based"
+    TokenBased = "Token Based"
+
+
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.clinical_type_col]
+Clinical_TYPES_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-
+DATASET_BENCHMARK_COLS = [t.value.col_name for t in HarnessTasks]
+TYPES_BENCHMARK_COLS = [t.value.col_name for t in ClinicalTypes]
+
+NUMERIC_INTERVALS = {
+    "?": pd.Interval(-1, 0, closed="right"),
+    "~1.5": pd.Interval(0, 2, closed="right"),
+    "~3": pd.Interval(2, 4, closed="right"),
+    "~7": pd.Interval(4, 9, closed="right"),
+    "~13": pd.Interval(9, 20, closed="right"),
+    "~35": pd.Interval(20, 45, closed="right"),
+    "~60": pd.Interval(45, 70, closed="right"),
+    "70+": pd.Interval(70, 10000, closed="right"),
+}

src/envs.py

@@ -4,22 +4,22 @@ from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+TOKEN = os.environ.get("TOKEN")  # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "m42-health"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+REPO_ID = f"{OWNER}/MEDIC-Benchmark"
+QUEUE_REPO = f"{OWNER}/medic-harness-requests"
+RESULTS_REPO = f"{OWNER}/medic-harness-results"
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
+CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "medic-harness-requests")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "medic-harness-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "medic-harness-requests-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "medic-harness-results-bk")
 
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -8,40 +8,48 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, ClinicalTypes
 from src.submission.check_validity import is_model_on_hub
 
 
 @dataclass
 class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run."""
+
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
+    org: str
     model: str
-    revision: str # commit hash, "" if main
-    results: dict
+    revision: str  # commit hash, "" if main
+    dataset_results: dict
+    # clinical_type_results:dict
     precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
+    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original  # Original or Adapter
+    architecture: str = "Unknown"
+    backbone:str = "Unknown"
     license: str = "?"
     likes: int = 0
     num_params: int = 0
-    date: str = "" # submission date of request file
+    date: str = ""  # submission date of request file
     still_on_hub: bool = False
+    display_result:bool = True
 
     @classmethod
-    def init_from_json_file(self, json_filepath):
+    def init_from_json_file(self, json_filepath, evaluation_metric):
         """Inits the result from the specific model result file"""
         with open(json_filepath) as fp:
             data = json.load(fp)
-
+ 
         config = data.get("config")
 
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
+        model_type = ModelType.from_str(config.get("model_type", ""))
+        license = config.get("license", "?")
+        num_params = config.get("num_params", "?")
+        display_result = config.get("display_result", True)
+        display_result = False if display_result=="False" else True
 
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
@@ -58,17 +66,17 @@ class EvalResult:
         full_model = "/".join(org_and_model)
 
         still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+            full_model, config.get("revision", "main"), trust_remote_code=True, test_tokenizer=False
         )
-        architecture = "?"
+        backbone = "?"
         if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
+            backbones = getattr(model_config, "architectures", None)
+            if backbones:
+                backbone = ";".join(backbones)
 
         # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
+        dataset_results = {}
+        for task in HarnessTasks:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
@@ -76,19 +84,37 @@ class EvalResult:
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
+            mean_acc = np.mean(accs)  # * 100.0
+            dataset_results[task.benchmark] = mean_acc
+        print(dataset_results)
+        # types_results = {}
+        # for clinical_type in ClinicalTypes:
+        #     clinical_type = clinical_type.value
+
+        #     # We average all scores of a given metric (not all metrics are present in all files)
+        #     accs = np.array([v.get(clinical_type.metric, None) for k, v in data[evaluation_metric]["clinical_type_results"].items() if clinical_type.benchmark == k])
+        #     if accs.size == 0 or any([acc is None for acc in accs]):
+        #         continue
+
+        #     mean_acc = np.mean(accs)  # * 100.0
+        #     types_results[clinical_type.benchmark] = mean_acc
 
         return self(
             eval_name=result_key,
             full_model=full_model,
             org=org,
             model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
+            dataset_results=dataset_results,
+            # clinical_type_results=types_results,
+            precision=precision,
+            revision=config.get("revision", ""),
             still_on_hub=still_on_hub,
-            architecture=architecture
+            # architecture=model_architecture,
+            backbone=backbone,
+            model_type=model_type,
+            num_params=num_params,
+            license=license,
+            display_result=display_result
         )
 
     def update_with_request_file(self, requests_path):
@@ -104,32 +130,66 @@ class EvalResult:
             self.likes = request.get("likes", 0)
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
+            # self.precision = request.get("precision", "float32")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+            pass
+            # print(
+            #     f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}"
+            # )
+            # print(f" Args used were - {request_file=}, {requests_path=}, {self.full_model=},")
 
-    def to_dict(self):
+    def to_dict(self, subset):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
+        if subset == "datasets":
+            average = sum([v for v in self.dataset_results.values() if v is not None]) / len(HarnessTasks)
+            data_dict = {
+                "eval_name": self.eval_name,  # not a column, just a save name,
+                AutoEvalColumn.precision.name: self.precision.value.name,
+                AutoEvalColumn.model_type.name: self.model_type.value.name,
+                AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+                AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+                # AutoEvalColumn.architecture.name: self.architecture.value.name,
+                AutoEvalColumn.backbone.name: self.backbone,
+                AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+                AutoEvalColumn.revision.name: self.revision,
+                AutoEvalColumn.average.name: average,
+                AutoEvalColumn.license.name: self.license,
+                AutoEvalColumn.likes.name: self.likes,
+                AutoEvalColumn.params.name: self.num_params,
+                AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+                "display_result" : self.display_result,
+            }
+
+            for task in HarnessTasks:
+                data_dict[task.value.col_name] = self.dataset_results[task.value.benchmark]
+
+            return data_dict
+        
+        if subset == "clinical_types":
+            average = sum([v for v in self.clinical_type_results.values() if v is not None]) / len(ClinicalTypes)
+            data_dict = {
+                "eval_name": self.eval_name,  # not a column, just a save name,
+                AutoEvalColumn.precision.name: self.precision.value.name,
+                AutoEvalColumn.model_type.name: self.model_type.value.name,
+                AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+                AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+                AutoEvalColumn.architecture.name: self.architecture.value.name,
+                AutoEvalColumn.backbone.name: self.backbone,
+                AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+                AutoEvalColumn.revision.name: self.revision,
+                AutoEvalColumn.average.name: average,
+                AutoEvalColumn.license.name: self.license,
+                AutoEvalColumn.likes.name: self.likes,
+                AutoEvalColumn.params.name: self.num_params,
+                AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+                "display_result" : self.display_result,
+            }
+
+            for clinical_type in ClinicalTypes:
+                data_dict[clinical_type.value.col_name] = self.clinical_type_results[clinical_type.value.benchmark]
+
+            return data_dict
+        
 
 
 def get_request_file_for_model(requests_path, model_name, precision):
@@ -146,15 +206,12 @@ def get_request_file_for_model(requests_path, model_name, precision):
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
+            if req_content["status"] in ["FINISHED"] and req_content["precision"] == precision.split(".")[-1]:
                 request_file = tmp_request_file
     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+def get_raw_eval_results(results_path: str, requests_path: str, evaluation_metric: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
 
@@ -175,20 +232,23 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
+        eval_result = EvalResult.init_from_json_file(model_result_filepath, evaluation_metric)
+        # eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
         eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
+        # if eval_name in eval_results.keys():
+        #     eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        # else:
+        eval_results[eval_name] = eval_result
 
     results = []
+    # clinical_type_results = []
     for v in eval_results.values():
         try:
-            v.to_dict() # we test if the dict version is complete
+            v.to_dict(subset="dataset")  # we test if the dict version is complete
+            if not v.display_result:
+                continue
             results.append(v)
         except KeyError:  # not all eval values present
             continue

src/populate.py

@@ -8,18 +8,21 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, evaluation_metric:str, subset:str) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
+    raw_data = get_raw_eval_results(results_path, requests_path, evaluation_metric)
+    # print(raw_data)
+    # raise Exception("stop")
+    all_data_json = [v.to_dict(subset=subset) for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    cols = list(set(df.columns).intersection(set(cols)))
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
+    return raw_data, df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
@@ -33,19 +36,19 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             with open(file_path) as fp:
                 data = json.load(fp)
 
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"])
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
 
             all_evals.append(data)
         elif ".md" not in entry:
             # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
             for sub_entry in sub_entries:
                 file_path = os.path.join(save_path, entry, sub_entry)
                 with open(file_path) as fp:
                     data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                # print(data)
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"])
                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
                 all_evals.append(data)
 

src/submission/check_validity.py

@@ -59,14 +59,24 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
         return False, "was not found on hub!", None
 
 
-def get_model_size(model_info: ModelInfo, precision: str):
+def get_model_size(model_info: ModelInfo, precision: str=None):
     """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    size_pattern = re.compile(r"(\d+\.)?\d+(b|m)")
+
     try:
         model_size = round(model_info.safetensors["total"] / 1e9, 3)
     except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+        try:
+            size_match = re.search(size_pattern, model_info.id.lower())
+            model_size = size_match.group(0)
+            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    if precision:
+        size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    else:
+        size_factor = 1
     model_size = size_factor * model_size
     return model_size
 
@@ -88,12 +98,12 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+                    file_names.append(f"{info['model_name']}_{info['revision']}")
 
                     # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                    if info["model_name"].count("/") == 0 or "submitted_time" not in info:
                         continue
-                    organisation, _ = info["model"].split("/")
+                    organisation, _ = info["model_name"].split("/")
                     users_to_submission_dates[organisation].append(info["submitted_time"])
 
     return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,5 +1,6 @@
 import json
 import os
+import ast
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
@@ -10,18 +11,57 @@ from src.submission.check_validity import (
     get_model_size,
     is_model_on_hub,
 )
+from src.display.utils import PromptTemplateName
 
 REQUESTED_MODELS = None
 USERS_TO_SUBMISSION_DATES = None
 
+PLACEHOLDER_DATASET_WISE_NORMALIZATION_CONFIG = """{
+    "NCBI" : {
+        "" : "condition"
+    },
+    "CHIA" : {
+        "" : "condition"
+        "" : "drug"
+        "" : "procedure"
+        "" : "measurement"
+    },
+    "BIORED" : {
+        "" : "condition"
+        "" : "drug"
+        "" : "gene"
+        "" : "gene variant"
+    },
+    "BC5CDR" : {
+        "" : "condition"
+        "" : "drug"
+    }
+}
+
+"""
+
 def add_new_eval(
     model: str,
-    base_model: str,
+    # base_model: str,
     revision: str,
-    precision: str,
-    weight_type: str,
+    # precision: str,
+    # weight_type: str,
+    model_arch: str,
+    label_normalization_map: str,
+    gliner_threshold:str,
+    gliner_tokenizer_bool:str,
+    prompt_template_name:str,
     model_type: str,
 ):
+    """
+    Saves request if valid else returns the error.
+    Validity is checked based on -
+        - model's existence on hub
+        - necessary info on the model's card
+        - label normalization is a valid python dict and contains the keys for all datasets
+        - threshold for gliner is a valid float
+
+    """
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
@@ -33,26 +73,35 @@ def add_new_eval(
         user_name = model.split("/")[0]
         model_path = model.split("/")[1]
 
-    precision = precision.split(" ")[0]
+    # precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
+    
+    model_type = model_type.split(":")[-1].strip()
 
     # Does the model actually exist?
     if revision == "":
         revision = "main"
 
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
+    # # Is the model on the hub?
+    # if weight_type in ["Delta", "Adapter"]:
+    #     base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+    #     if not base_model_on_hub:
+    #         return styled_error(f'Base model "{base_model}" {error}')
 
-    if not weight_type == "Adapter":
+    if not model_arch == "GLiNER Encoder":
         model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
+    else:
+        model_name_matches = list(API.list_models(model_name=model))
+        if len(model_name_matches) < 1:
+            return styled_error(f'Model "{model}" does not exist on the hub!')   
+        elif model_name_matches[0].id != model:
+            return styled_error(f'Model "{model}" does not exist on the hub! There might be a typo in the name')            
+    
 
     # Is the model info correctly filled?
     try:
@@ -60,7 +109,7 @@ def add_new_eval(
     except Exception:
         return styled_error("Could not get your model information. Please fill it up properly.")
 
-    model_size = get_model_size(model_info=model_info, precision=precision)
+    model_size = get_model_size(model_info=model_info)
 
     # Were the model card and license filled?
     try:
@@ -72,32 +121,71 @@ def add_new_eval(
     if not modelcard_OK:
         return styled_error(error_msg)
 
+    # Verify the inference config now
+    try:
+        label_normalization_map = ast.literal_eval(label_normalization_map)
+    except Exception as e:
+        return styled_error("Please enter a valid json for the labe; normalization map")
+
+    inference_config = {
+        # "model_arch" : model_arch,
+        "label_normalization_map": label_normalization_map,
+    }
+
+    match model_arch:
+        case "Encoder":
+            pass
+        case "Decoder":
+            if not prompt_template_name in [prompt_template.value for prompt_template in PromptTemplateName]:
+                return styled_error("Prompt template name is invalid")
+            inference_config = {
+                **inference_config,
+                "prompt_template_identifier": prompt_template_name,
+            }
+        case "GLiNER Encoder":
+                try:
+                    gliner_threshold = float(gliner_threshold)
+                    gliner_tokenizer_bool = ast.literal_eval(gliner_tokenizer_bool)
+                    inference_config = {
+                        **inference_config,
+                        "gliner_threshold": gliner_threshold,
+                        "gliner_tokenizer_bool" : gliner_tokenizer_bool
+                    }
+                except Exception as e:
+                    return styled_error("Please enter a valid float for the threshold")
+        case _:
+            return styled_error("Model Architecture is invalid")
+
     # Seems good, creating the eval
     print("Adding new eval")
 
+
     eval_entry = {
-        "model": model,
-        "base_model": base_model,
+        "model_name": model,
+        # "base_model": base_model,
         "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
+        # "precision": precision,
+        # "weight_type": weight_type,
+        "model_architecture": model_arch,
         "status": "PENDING",
         "submitted_time": current_time,
         "model_type": model_type,
         "likes": model_info.likes,
-        "params": model_size,
+        "num_params": model_size,
         "license": license,
         "private": False,
+        "inference_config":inference_config,
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
+
+    if f"{model}_{revision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted. Add the revision if the model has been updated.")
 
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    out_path = f"{OUT_DIR}/{model_path}_{revision}_eval_request.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))