add open-source

app.py

@@ -65,13 +65,15 @@ def init_leaderboard(dataframe):
         datatype=[c.type for c in fields(AutoEvalColumn)],
         select_columns=SelectColumns(
             default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.never_displayed],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.generation_model.name, AutoEvalColumn.retrieval_model.name],
+        search_columns=[AutoEvalColumn.generative_model_link.name, AutoEvalColumn.retrieval_model_link.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
             ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn.retrieval_model.name, type="checkboxgroup", label="Retrieval models"),
+            ColumnFilter(AutoEvalColumn.generative_model.name, type="checkboxgroup", label="Generative models"),
             # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
             # ColumnFilter(
             #     AutoEvalColumn.params.name,

eval-results/demo-leaderboard/CLOSE_deepseek-v2-chat/results_2023-12-08 15:46:20.425378.json

@@ -19,16 +19,16 @@
   },
   "config": {
     "eval_name": "CLOSE_deepseek-v2-chat",
-    "generation_model": "deepseek-ai/DeepSeek-V2-Chat-0628",
-    "generation_model_args": {
+    "generative_model": "deepseek-ai/DeepSeek-V2-Chat-0628",
+    "generative_model_args": {
       "name": "deepseek-ai/DeepSeek-V2-Chat-0628",
-      "num_params": 80,
+      "num_params": 236,
       "open_source": true
     },
     "retrieval_model": "CLOSE",
     "retrieval_model_args": {
       "num_params": 0.0,
-      "open_source": false
+      "open_source": true
     }
   }
 }

eval-results/demo-leaderboard/CLOSE_llama3-70b-instruct/results_2023-12-08 15:46:20.425378.json

@@ -19,16 +19,16 @@
   },
   "config": {
     "eval_name": "CLOSE_llama3-70b-instruct",
-    "generation_model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
-    "generation_model_args": {
+    "generative_model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+    "generative_model_args": {
       "name": "meta-llama/Meta-Llama-3.1-70B-Instruct",
-      "num_params": 70,
+      "num_params": 70.6,
       "open_source": true
     },
     "retrieval_model": "CLOSE",
     "retrieval_model_args": {
       "num_params": 0.0,
-      "open_source": false
+      "open_source": true
     }
   }
 }

eval-results/demo-leaderboard/CLOSE_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -19,16 +19,16 @@
   },
   "config": {
     "eval_name": "CLOSE_qwen2-72b",
-    "generation_model": "Qwen/Qwen2.5-72B-Instruct",
-    "generation_model_args": {
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
       "name": "Qwen/Qwen2.5-72B-Instruct",
-      "num_params": 72,
+      "num_params": 72.7,
       "open_source": true
     },
     "retrieval_model": "CLOSE",
     "retrieval_model_args": {
       "num_params": 0.0,
-      "open_source": false
+      "open_source": true
     }
   }
 }

eval-results/demo-leaderboard/CLOSE_yi15-34b/results_2023-12-08 15:46:20.425378.json

@@ -19,16 +19,16 @@
   },
   "config": {
     "eval_name": "CLOSE_yi15-34b",
-    "generation_model": "01ai/Yi-1.5-34B-Chat-16K",
-    "generation_model_args": {
+    "generative_model": "01ai/Yi-1.5-34B-Chat-16K",
+    "generative_model_args": {
       "name": "01ai/Yi-1.5-34B-Chat-16K",
-      "num_params": 34,
+      "num_params": 34.4,
       "open_source": true
     },
     "retrieval_model": "CLOSE",
     "retrieval_model_args": {
       "num_params": 0.0,
-      "open_source": false
+      "open_source": true
     }
   }
 }

eval-results/demo-leaderboard/gte-qwen2-1.5b_deepseek-v2-chat/results_2023-12-08 15:46:20.425378.json

@@ -19,16 +19,16 @@
   },
   "config": {
     "eval_name": "gte-qwen2-1.5b_deepseek-v2-chat",
-    "generation_model": "deepseek-ai/DeepSeek-V2-Chat-0628",
-    "generation_model_args": {
+    "generative_model": "deepseek-ai/DeepSeek-V2-Chat-0628",
+    "generative_model_args": {
       "name": "deepseek-ai/DeepSeek-V2-Chat-0628",
-      "num_params": 80,
+      "num_params": 236,
       "open_source": true
     },
     "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
     "retrieval_model_args": {
       "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
-      "num_params": 1.5,
+      "num_params": 1.78,
       "open_source": true
     }
   }

eval-results/demo-leaderboard/gte-qwen2-1.5b_llama3-70b-instruct/results_2023-12-08 15:46:20.425378.json

@@ -19,16 +19,16 @@
   },
   "config": {
     "eval_name": "gte-qwen2-1.5b_llama3-70b-instruct",
-    "generation_model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
-    "generation_model_args": {
+    "generative_model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+    "generative_model_args": {
       "name": "meta-llama/Meta-Llama-3.1-70B-Instruct",
-      "num_params": 70,
+      "num_params": 70.6,
       "open_source": true
     },
     "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
     "retrieval_model_args": {
       "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
-      "num_params": 1.5,
+      "num_params": 1.78,
       "open_source": true
     }
   }

eval-results/demo-leaderboard/gte-qwen2-1.5b_qwen2-72b/results_2023-12-08 15:46:20.425378.json

@@ -19,16 +19,16 @@
   },
   "config": {
     "eval_name": "gte-qwen2-1.5b_qwen2-72b",
-    "generation_model": "Qwen/Qwen2.5-72B-Instruct",
-    "generation_model_args": {
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
       "name": "Qwen/Qwen2.5-72B-Instruct",
-      "num_params": 72,
+      "num_params": 72.7,
       "open_source": true
     },
     "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
     "retrieval_model_args": {
       "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
-      "num_params": 1.5,
+      "num_params": 1.78,
       "open_source": true
     }
   }

eval-results/demo-leaderboard/gte-qwen2-1.5b_yi15-34b/results_2023-12-08 15:46:20.425378.json

@@ -19,16 +19,16 @@
   },
   "config": {
     "eval_name": "gte-qwen2-1.5b_yi15-34b",
-    "generation_model": "01ai/Yi-1.5-34B-Chat-16K",
-    "generation_model_args": {
+    "generative_model": "01ai/Yi-1.5-34B-Chat-16K",
+    "generative_model_args": {
       "name": "01ai/Yi-1.5-34B-Chat-16K",
-      "num_params": 34,
+      "num_params": 34.4,
       "open_source": true
     },
     "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
     "retrieval_model_args": {
       "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
-      "num_params": 1.5,
+      "num_params": 1.78,
       "open_source": true
     }
   }

eval-results/demo-leaderboard/qwen2-72b_bge-large-zh/results_2023-12-08 15:46:20.425378.json

@@ -18,17 +18,17 @@
     }
   },
   "config": {
-    "eval_name": "bge-large-zh_bge-large-zh",
-    "generation_model": "BAAI/bge-large-zh",
-    "generation_model_args": {
-      "name": "BAAI/bge-large-zh",
-      "num_params": 0.2,
+    "eval_name": "qwen2-72b_bge-large-zh",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
       "open_source": true
     },
     "retrieval_model": "BAAI/bge-large-zh",
     "retrieval_model_args": {
       "name": "BAAI/bge-large-zh",
-      "num_params": 0.2,
+      "num_params": 0.326,
       "open_source": true
     }
   }

eval-results/demo-leaderboard/qwen2-72b_bge-m3/results_2023-12-08 15:46:20.425378.json

@@ -18,17 +18,17 @@
     }
   },
   "config": {
-    "eval_name": "bge-m3_bge-m3",
-    "generation_model": "BAAI/bge-m3",
-    "generation_model_args": {
-      "name": "BAAI/bge-m3",
-      "num_params": 0.2,
+    "eval_name": "qwen2-72b_bge-m3",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
       "open_source": true
     },
     "retrieval_model": "BAAI/bge-m3",
     "retrieval_model_args": {
       "name": "BAAI/bge-m3",
-      "num_params": 0.2,
+      "num_params": 0.5,
       "open_source": true
     }
   }

eval-results/demo-leaderboard/qwen2-72b_e5-mistral-7b/results_2023-12-08 15:46:20.425378.json

@@ -18,17 +18,17 @@
     }
   },
   "config": {
-    "eval_name": "e5-mistral-7b_e5-mistral-7b",
-    "generation_model": "intfloat/e5-mistral-7b-instruct",
-    "generation_model_args": {
-      "name": "intfloat/e5-mistral-7b-instruct",
-      "num_params": 7,
+    "eval_name": "qwen2-72b_e5-mistral-7b",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
       "open_source": true
     },
     "retrieval_model": "intfloat/e5-mistral-7b-instruct",
     "retrieval_model_args": {
       "name": "intfloat/e5-mistral-7b-instruct",
-      "num_params": 7,
+      "num_params": 7.11,
       "open_source": true
     }
   }

eval-results/demo-leaderboard/qwen2-72b_gte-qwen2-1.5b/results_2023-12-08 15:46:20.425378.json

@@ -18,17 +18,17 @@
     }
   },
   "config": {
-    "eval_name": "gte-qwen2-1.5b_gte-qwen2-1.5b",
-    "generation_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
-    "generation_model_args": {
-      "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
-      "num_params": 1.5,
+    "eval_name": "qwen2-72b_gte-qwen2-1.5b",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
       "open_source": true
     },
     "retrieval_model": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
     "retrieval_model_args": {
       "name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
-      "num_params": 1.5,
+      "num_params": 1.78,
       "open_source": true
     }
   }

eval-results/demo-leaderboard/qwen2-72b_jina-zh/results_2023-12-08 15:46:20.425378.json

@@ -18,17 +18,17 @@
     }
   },
   "config": {
-    "eval_name": "jina-zh_jina-zh",
-    "generation_model": "jinaai/reader-lm-0.5b",
-    "generation_model_args": {
-      "name": "jinaai/reader-lm-0.5b",
-      "num_params": 0.2,
+    "eval_name": "qwen2-72b_jina-zh",
+    "generative_model": "Qwen/Qwen2.5-72B-Instruct",
+    "generative_model_args": {
+      "name": "Qwen/Qwen2.5-72B-Instruct",
+      "num_params": 72.7,
       "open_source": true
     },
-    "retrieval_model": "jinaai/reader-lm-0.5b",
+    "retrieval_model": "jinaai/jina-embeddings-v2-base-zh",
     "retrieval_model_args": {
-      "name": "jinaai/reader-lm-0.5b",
-      "num_params": 0.2,
+      "name": "jinaai/jina-embeddings-v2-base-zh",
+      "num_params": 0.161,
       "open_source": true
     }
   }

src/about.py

@@ -16,21 +16,21 @@ class Tasks(Enum):
     # task1 = Task("logiqa", "acc_norm", "LogiQA")
 
     # retrieval tasks
-    mrr = Task("retrieval", "mrr", "MRR")
-    map = Task("retrieval", "map", "MAP")
+    mrr = Task("retrieval", "mrr", "MRR ⬆️")
+    map = Task("retrieval", "map", "MAP ⬆️")
 
     # generation tasks
-    em = Task("generation", "em", "EM")
-    f1 = Task("generation", "f1", "F1")
-    rouge1 = Task("generation", "rouge1", "Rouge-1")
-    rouge2 = Task("generation", "rouge2", "Rouge-2")
-    rougeL = Task("generation", "rougeL", "Rouge-L")
+    em = Task("generation", "em", "EM ⬆️")
+    f1 = Task("generation", "f1", "F1 ⬆️")
+    rouge1 = Task("generation", "rouge1", "Rouge-1 ⬆️")
+    rouge2 = Task("generation", "rouge2", "Rouge-2 ⬆️")
+    rougeL = Task("generation", "rougeL", "Rouge-L ⬆️")
 
-    accuracy = Task("generation", "accuracy", "ACC")
-    completeness = Task("generation", "completeness", "COMP")
-    hallucination = Task("generation", "hallucination", "HAL")
-    utilization = Task("generation", "utilization", "UTIL")
-    numerical_accuracy = Task("generation", "numerical_accuracy", "MACC")
+    accuracy = Task("generation", "accuracy", "ACC ⬆️")
+    completeness = Task("generation", "completeness", "COMP ⬆️")
+    hallucination = Task("generation", "hallucination", "HAL ⬇️")
+    utilization = Task("generation", "utilization", "UTIL ⬆️")
+    numerical_accuracy = Task("generation", "numerical_accuracy", "MACC ⬆️")
 
 
 NUM_FEWSHOT = 0 # Change with your few shot
@@ -39,19 +39,121 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Fin Benchmark leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">🏅 OmniEval Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+<div align="center">OmniEval: Omnidirectional and Automatic RAG Evaluation Benchmark in Financial Domain</div>
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## How it works
+# <div align="center">OmniEval: Omnidirectional and Automatic RAG Evaluation Benchmark in Financial Domain</div>
+
+
+<div align="center">
+<!-- <a href="https://arxiv.org/abs/2405.13576" target="_blank"><img src=https://img.shields.io/badge/arXiv-b5212f.svg?logo=arxiv></a> -->
+<!-- <a href="https://huggingface.co/datasets/RUC-NLPIR/FlashRAG_datasets/" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace%20Datasets-27b3b4.svg></a> -->
+<!-- <a href="https://huggingface.co/ShootingWong/OmniEval-ModelEvaluator" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace%20Checkpoint-5fc372.svg></a> -->
+<!-- <a href="https://huggingface.co/ShootingWong/OmniEval-HallucinationEvaluator" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace%20Checkpoint-b181d9.svg></a> -->
+<a href="https://huggingface.co/datasets/RUC-NLPIR/FlashRAG_datasets/" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Dataset-27b3b4></a>
+<a href="https://huggingface.co/ShootingWong/OmniEval-ModelEvaluator" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Checkpoint-5fc372></a>
+<a href="https://huggingface.co/ShootingWong/OmniEval-HallucinationEvaluator" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Checkpoint-b181d9></a>
+<a href="https://huggingface.co/spaces/NLPIR-RAG/OmniEval" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Leaderboard-blue></a>
+<a href="https://github.com/RUC-NLPIR/FlashRAG/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/badge/LICENSE-MIT-green"></a>
+<a><img alt="Static Badge" src="https://img.shields.io/badge/made_with-Python-blue"></a>
+</div>
+
+<!-- [![HuggingFace](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Leaderboard-blue)](https://huggingface.co/spaces/Vchitect/VBench_Leaderboard) -->
+
+<h4 align="center">
+
+<p>
+<a href="#wrench-installation">Installation</a> |
+<!-- <a href="#sparkles-features">Features</a> | -->
+<a href="#rocket-quick-start">Quick-Start</a> | 
+<a href="#bookmark-license">License</a> | 
+<a href="#star2-citation">Citation</a>
+
+</p>
+
+</h4>
+
+<!-- 
+With FlashRAG and provided resources, you can effortlessly reproduce existing SOTA works in the RAG domain or implement your custom RAG processes and components. -->
+
+
+## :wrench: Installation
+`conda env create -f environment.yml && conda activate finrag`
+
+<!-- ## :sparkles: Features
+1. -->
+## :rocket: Quick-Start
+Notion:
+1. The code run path is `./OpenFinBench`
+2. We provide our auto-generated evaluation dataset in <a href="https://huggingface.co/datasets/RUC-NLPIR/FlashRAG_datasets/" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Dataset-27b3b4></a>
+### 1. Build the Retrieval Corpus
+```
+# cd OpenFinBench
+sh corpus_builder/build_corpus.sh # Please see the annotation inner the bash file to set parameters.
+```
+### 2. Generate Evaluation Data Samples
+1. Generate evaluation instances
+```
+# cd OpenFinBench
+sh data_generator/generate_data.sh
+```
+2. Filter (quality inspection) evaluation instances
+```
+sh data_generator/generate_data_filter.sh
+```
+### 3. Inference Your Models
+```
+# cd OpenFinBench
+sh evaluator/inference/rag_inference.sh 
+```
+### 4. Evaluate Your Models
+#### (a) Rule-based Evaluation
+```
+# cd OpenFinBench
+sh evaluator/judgement/judger.sh # by setting judge_type="rule"
+```
+#### (b) Model-based Evalution
+We propose five model-based metric: accuracy, completeness, utilization, numerical_accuracy, and hallucination. We have trained two models from Qwen2.5-7B by the lora strategy and human-annotation labels to implement model-based evaluation. 
+
+Note that the evaluator of hallucination is different from other four. Their model checkpoint can be load from the following huggingface links:
+1. The evaluator for hallucination metric: <a href="https://huggingface.co/ShootingWong/OmniEval-HallucinationEvaluator" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Checkpoint-b181d9></a>
+2. The evaluator for other metric: <a href="https://huggingface.co/ShootingWong/OmniEval-ModelEvaluator" target="_blank"><img src=https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Checkpoint-5fc372></a>
+
+
+
+To implement model-based evaluation, you can first set up two vllm servers by the following codes:
+```
+```
+
+Then conduct the model-based evaluate using the following codes, (change the parameters inner the bash file).
+```
+sh evaluator/judgement/judger.sh 
+``` 
+
+## :bookmark: License
+
+OmniEval is licensed under the [<u>MIT License</u>](./LICENSE).
+
+## :star2: Citation
+The paper is waiting to be released!
+
+<!-- # Check Infos
+## Pipeline
+1. Build corpus
+2. Data generation
+3. RAG inference
+4. Result evaluatioin
+
+## Code
+1. remove "baichuan"
+2. remove useless annotation -->
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
 
 """
 

src/display/formatting.py

@@ -5,6 +5,8 @@ def model_hyperlink(link, model_name):
 def make_clickable_model(model_name, model_link=None):
     if model_link:
         return model_hyperlink(model_link, model_name)
+    if model_name == "CLOSE":
+        return model_name
     link = f"https://huggingface.co/{model_name}"
     return model_hyperlink(link, model_name)
 

src/display/utils.py

@@ -17,17 +17,21 @@ class ColumnContent:
     displayed_by_default: bool
     hidden: bool = False
     never_hidden: bool = False
+    never_displayed: bool = False
 
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("Type Symbol", "str", True, never_hidden=True)])
 # auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-auto_eval_column_dict.append(["retrieval_model", ColumnContent, ColumnContent("Retrieval Model", "markdown", True, never_hidden=True)])
-auto_eval_column_dict.append(["generation_model", ColumnContent, ColumnContent("Generation Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["retrieval_model", ColumnContent, ColumnContent("Retrieval Model Plain", "markdown", False, never_displayed=True)])
+auto_eval_column_dict.append(["generative_model", ColumnContent, ColumnContent("Generative Model Plain", "markdown", False, never_displayed=True)])
+auto_eval_column_dict.append(["retrieval_model_link", ColumnContent, ColumnContent("Retrieval Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["generative_model_link", ColumnContent, ColumnContent("Generative Model", "markdown", True, never_hidden=True)])
 
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["gen_average", ColumnContent, ColumnContent("Gen Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["ret_average", ColumnContent, ColumnContent("Ret Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 

src/leaderboard/read_evals.py

@@ -18,12 +18,12 @@ class EvalResult:
     """
     eval_name: str # org_model_precision (uid)
     org: str 
-    generation_model: str
+    generative_model: str
     retrieval_model: str
     # revision: str # commit hash, "" if main
     results: dict
-    generation_model_link: str = "" # link to the model on the hub
-    generation_model_args: dict = None
+    generative_model_link: str = "" # link to the model on the hub
+    generative_model_args: dict = None
     retrieval_model_link: str = "" # link to the model on the hub
     retrieval_model_args: dict = None
     precision: Precision = Precision.Unknown
@@ -48,7 +48,7 @@ class EvalResult:
 
         # Get model and org
         eval_name= config.get("eval_name", "")
-        generation_model = config.get("generation_model", "")
+        generative_model = config.get("generative_model", "")
         retrieval_model = config.get("retrieval_model", "")
         org= config.get("org", "")
         # org_and_model = org_and_model.split("/", 1)
@@ -77,17 +77,21 @@ class EvalResult:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k and task.col_name != "hallucination"])
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
             mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc
+            results[task.metric] = data["results"][task.benchmark].get(task.metric, None)
+            if results[task.metric] is not None:
+                results[task.metric] = results[task.metric] * 100.0
 
-        generation_model_args = config.get("generation_model_args", None)
+
+        generative_model_args = config.get("generative_model_args", None)
         retrieval_model_args = config.get("retrieval_model_args", None)
         open_source= True
-        if not generation_model_args or not generation_model_args.get("open_source", False):
+        if not generative_model_args or not generative_model_args.get("open_source", False):
             open_source = False
         if not retrieval_model_args or not retrieval_model_args.get("open_source", False):
             open_source = False
@@ -96,10 +100,10 @@ class EvalResult:
             eval_name=eval_name,
             # full_model=full_model,
             org=org,
-            generation_model=generation_model,
+            generative_model=generative_model,
             retrieval_model=retrieval_model,
             results=results,
-            generation_model_args=generation_model_args,
+            generative_model_args=generative_model_args,
             retrieval_model_args=retrieval_model_args,
             model_type=ModelType.OpenSource if open_source else ModelType.ClosedSource,
             # precision=precision,
@@ -126,7 +130,6 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             # AutoEvalColumn.precision.name: self.precision.value.name,
@@ -134,20 +137,23 @@ class EvalResult:
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             # AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.generation_model.name: make_clickable_model(self.generation_model, self.generation_model_link),
-            AutoEvalColumn.retrieval_model.name: make_clickable_model(self.retrieval_model, self.retrieval_model_link),
+            AutoEvalColumn.generative_model.name: self.generative_model,
+            AutoEvalColumn.retrieval_model.name: self.retrieval_model,
+            AutoEvalColumn.generative_model_link.name: make_clickable_model(self.generative_model, self.generative_model_link),
+            AutoEvalColumn.retrieval_model_link.name: make_clickable_model(self.retrieval_model, self.retrieval_model_link),
             # AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.ret_average.name: self.results["retrieval"],
+            AutoEvalColumn.gen_average.name: self.results["generation"],
             # AutoEvalColumn.license.name: self.license,
             # AutoEvalColumn.likes.name: self.likes,
-            # AutoEvalColumn.generation_model_params.name: self.num_params,
+            # AutoEvalColumn.generative_model_params.name: self.num_params,
             # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-            "Gen#Params (B)": self.generation_model_args.get("num_params", "Unknown"),
+            "Gen#Params (B)": self.generative_model_args.get("num_params", "Unknown"),
             "Ret#Params (B)": self.retrieval_model_args.get("num_params", "Unknown"),
         }
 
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+            data_dict[task.value.col_name] = self.results[task.value.metric]
 
         return data_dict
 

src/populate.py

@@ -15,8 +15,10 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     print(all_data_json)
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df.sort_values(by=[AutoEvalColumn.gen_average.name], ascending=False)
     df = df[cols].round(decimals=2)
+    # assert column 'Retrieval Model Plain' in df.columns
+    # assert AutoEvalColumn.retrieval_model.name in df.columns, f"Column {df.columns} does not contain 'Retrieval Model Plain'"
 
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]