Upload 6 files

Inaccessible_model.csv

@@ -0,0 +1,3 @@
+Model,Language Model,Open Source,Text Recognition,Scene Text-Centric VQA,Doc-Oriented VQA,KIE,HMER,Final Score,Link
+Nanbeige-VL,-,No,260,173,117,137,40,727,https://github.com/jmiemirza/MMFM-Challenge/blob/master/2024-winnners-reports
+BlueLM-VL,-,No,209,173,132,147,6,667,https://github.com/vivo-ai-lab/BlueLM

OCRBench.csv

@@ -0,0 +1,29 @@
+Model,Language Model,Open Source,Text Recognition,Scene Text-Centric VQA,Doc-Oriented VQA,KIE,HMER,Final Score,Link
+Minicpm-V 2.6,Qwen2-7B,Yes, 261,186,176,181,48,852,https://github.com/OpenBMB/MiniCPM-V
+MiniMonkey,internlm2-chat-1_8b,Yes, 251,174,141,169,71,806,https://arxiv.org/abs/2408.02034
+H2OVL-Mississippi-2B,H2O-Danube2-1.8B,Yes,252,171,140,166,53,782,https://huggingface.co/h2oai/h2ovl-mississippi-2b
+InternVL2-1B,Qwen2-0.5B-Instruct,Yes, 255,166,130,156,72,779,https://huggingface.co/OpenGVLab/InternVL2-1B
+InternVL2-4B,Phi-3-mini-128k-instruct,Yes, 235,170,138,164,69,776,https://huggingface.co/OpenGVLab/InternVL2-4B
+InternVL2-2B,internlm2-chat-1_8b,Yes, 245,172,122,167,62,768,https://huggingface.co/OpenGVLab/InternVL2-2B
+H2OVL-Mississippi-0.8B,H2O-Danube3-0.5B,Yes, 274,162,112,152,51,751,https://huggingface.co/h2oai/h2ovl-mississippi-800m
+Qwen-VL-Max,-,No,254,166,148,143,12,723,https://github.com/QwenLM/Qwen-VL
+Qwen-VL-Plus,-,No,248,155,141,141,9,694,https://github.com/QwenLM/Qwen-VL
+Gemini,-,No,215,174,128,134,8,659,https://deepmind.google/technologies/gemini/
+GPT4V,-,No,167,163,146,160,9,645,https://openai.com/
+MiniCPM-V-2,MiniCPM-2.4B, Yes,245,171,103,86,0,605,https://github.com/OpenBMB/MiniCPM-V
+mPLUG-DocOwl1.5,LLaMA-2 7B, Yes,182,157,126,134,0,599,https://arxiv.org/abs/2403.12895
+TextMonkey,Qwen-7B,Yes,169,164,115,113,0,561,https://export.arxiv.org/abs/2403.04473
+InternVL-Chat-Chinese,LLaMA2-13B,Yes,228,153,72,64,0,517,https://arxiv.org/abs/2312.14238
+Monkey,Qwen-7B,Yes,174,161,91,88,0,514,https://arxiv.org/abs/2311.06607
+InternLM-XComposer2,InternLM2-7B,Yes,160,160,103,87,1,511,https://arxiv.org/abs/2401.16420
+QwenVL,Qwen-7B,Yes,179,157,95,75,0,506,https://arxiv.org/abs/2308.12966
+mPLUG-Owl2,LLaMA2-7B,Yes,153,153,41,19,0,366,https://arxiv.org/abs/2311.04257
+LLaVAR,LLaMA-13B.,Yes,186,122,25,13,0,346,https://arxiv.org/abs/2306.17107
+LLaVA1.5-13B,Vicuna-v1.5-13B,Yes,176,129,19,7,0,331,https://arxiv.org/abs/2310.03744
+InternLM-XComposer,InternLM-7B,Yes,192,91,14,6,0,303,https://arxiv.org/abs/2309.15112
+LLaVA1.5-7B,Vicuna-v1.5-7B,Yes,160,117,15,5,0,297,https://arxiv.org/abs/2310.03744
+mPLUG-Owl,LLaMA-2 7B,Yes,172,104,18,3,0,297,https://arxiv.org/abs/2304.14178
+BLIVA,Vicuna-7B,Yes,165,103,22,1,0,291,https://arxiv.org/abs/2308.09936
+InstructBLIP,Vicuna-7b,Yes,168,93,14,1,0,276,https://arxiv.org/abs/2305.06500
+BLIP2-6.7B,OPT-6.7B,Yes,154,71,10,0,0,235,https://arxiv.org/abs/2301.12597
+MiniGPT4V2,LLaMA2-13B,Yes,124,29,4,0,0,157,https://arxiv.org/abs/2310.09478

README.md

@@ -1,10 +1,13 @@
 ---
-title: OCRBench V2 Leaderboard
-emoji: 🐠
-colorFrom: red
-colorTo: yellow
-sdk: static
+title: Ocrbench Leaderboard
+emoji: 🏆
+colorFrom: gray
+colorTo: pink
+sdk: gradio
+sdk_version: 4.15.0
+app_file: app.py
 pinned: false
+license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

TextRecognition.csv

@@ -0,0 +1,26 @@
+Model,Language Model,Open Source,Regular Text,Irregular Text,Artistic Text,Handwriting,Digit string,Non-semantic Text,ALL,Link
+InternVL2-1B,Qwen2-0.5B-Instruct,Yes, 47,45,44,26,49,44,255,https://huggingface.co/OpenGVLab/InternVL2-1B
+InternVL2-4B,Phi-3-mini-128k-instruct,Yes,46,44,44,17,43,41,235,https://huggingface.co/OpenGVLab/InternVL2-4B
+InternVL2-2B,internlm2-chat-1_8b,Yes,46,45,45,26,42,41,245,https://huggingface.co/OpenGVLab/InternVL2-2B
+Nanbeige-VL,-,No,47,46,45,38,39,45,260,https://github.com/jmiemirza/MMFM-Challenge/blob/master/2024-winnners-reports
+Qwen-VL-Max,-,No,49,50,49,27,36,43,254,https://github.com/QwenLM/Qwen-VL
+Qwen-VL-Plus,-,No,49,49,48,36,23,43,248,https://github.com/QwenLM/Qwen-VL
+BlueLM-VL,-,No,46,39,42,20,27,35,209,https://github.com/vivo-ai-lab/BlueLM
+Gemini,-,No,47,35,45,31,25,32,215,https://deepmind.google/technologies/gemini/
+GPT4V,-,No,39,37,41,11,1,38,167,https://openai.com/
+mPLUG-DocOwl1.5,LLaMA-2 7B, Yes,45,39,39,22,15,22,182,https://arxiv.org/abs/2403.12895
+TextMonkey,Qwen-7B,Yes,45,35,39,15,9,26,169,https://export.arxiv.org/abs/2403.04473
+InternVL-Chat-Chinese,LLaMA2-13B,Yes,49,46,46,28,27,32,228,https://arxiv.org/abs/2312.14238
+Monkey,Qwen-7B,Yes,44,37,40,14,11,28,174,https://arxiv.org/abs/2311.06607
+InternLM-XComposer2,InternLM2-7B,Yes,45,37,37,12,7,22,160,https://arxiv.org/abs/2401.16420
+QwenVL,Qwen-7B,Yes,46,39,42,14,10,28,179,https://arxiv.org/abs/2308.12966
+mPLUG-Owl2,LLaMA2-7B,Yes,43,37,40,12,4,17,153,https://arxiv.org/abs/2311.04257
+LLaVAR,LLaMA-13B.,Yes,48,42,43,28,12,13,186,https://arxiv.org/abs/2306.17107
+LLaVA1.5-13B,Vicuna-v1.5-13B,Yes,48,44,43,30,7,4,176,https://arxiv.org/abs/2310.03744
+InternLM-XComposer,InternLM-7B,Yes,49,44,46,23,13,17,192,https://arxiv.org/abs/2309.15112
+LLaVA1.5-7B,Vicuna-v1.5-7B,Yes,43,40,41,26,5,5,160,https://arxiv.org/abs/2310.03744
+mPLUG-Owl,LLaMA-2 7B,Yes,44,42,44,13,9,20,172,https://arxiv.org/abs/2304.14178
+BLIVA,Vicuna-7B,Yes,48,42,42,24,5,4,165,https://arxiv.org/abs/2308.09936
+InstructBLIP,Vicuna-7b,Yes,46,43,44,19,8,8,168,https://arxiv.org/abs/2305.06500
+BLIP2-6.7B,OPT-6.7B,Yes,47,41,44,15,1,6,154,https://arxiv.org/abs/2301.12597
+MiniGPT4V2,LLaMA2-13B,Yes,35,37,36,13,1,2,124,https://arxiv.org/abs/2310.09478

app.py

@@ -0,0 +1,310 @@
+import ast
+import argparse
+import glob
+import pickle
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+block_css = """
+#notice_markdown {
+    font-size: 104%
+}
+#notice_markdown th {
+    display: none;
+}
+#notice_markdown td {
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#leaderboard_markdown {
+    font-size: 104%
+}
+#leaderboard_markdown td {
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#leaderboard_dataframe td {
+    line-height: 0.1em;
+}
+footer {
+    display:none !important
+}
+.image-container {
+    display: flex;
+    align-items: center;
+    padding: 1px;
+}
+.image-container img {
+    margin: 0 30px;
+    height: 20px;
+    max-height: 100%;
+    width: auto;
+    max-width: 20%;
+}
+"""
+def model_hyperlink(model_name, link):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+def load_leaderboard_table_csv(filename, add_hyperlink=True):
+    lines = open(filename).readlines()
+    heads = [v.strip() for v in lines[0].split(",")]
+    rows = []
+    for i in range(1, len(lines)):
+        row = [v.strip() for v in lines[i].split(",")]
+        for j in range(len(heads)):
+            item = {}
+            for h, v in zip(heads, row):
+                if h != "Model" and h != "Link" and h != "Language Model" and h != "Open Source":
+                    item[h] = int(v)
+                else:
+                    item[h] = v
+            if add_hyperlink:
+                item["Model"] = model_hyperlink(item["Model"], item["Link"])
+        rows.append(item)
+    return rows
+
+def get_arena_table(model_table_df):
+    # sort by rating
+    model_table_df = model_table_df.sort_values(by=["Final Score"], ascending=False)
+    values = []
+    for i in range(len(model_table_df)):
+        row = []
+        model_key = model_table_df.index[i]
+        model_name = model_table_df["Model"].values[model_key]
+        # rank
+        row.append(i + 1)
+        # model display name
+        row.append(model_name)
+
+        row.append(
+            model_table_df["Language Model"].values[model_key]
+        )
+        row.append(
+            model_table_df["Open Source"].values[model_key]
+        )
+        row.append(
+            model_table_df["Text Recognition"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["Scene Text-Centric VQA"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["Doc-Oriented VQA"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["KIE"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["HMER"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["Final Score"].values[model_key]
+        )
+        values.append(row)
+    return values
+
+def get_recog_table(model_table_df):
+    # sort by rating
+    values = []
+    for i in range(len(model_table_df)):
+        row = []
+        model_key = model_table_df.index[i]
+        model_name = model_table_df["Model"].values[model_key]
+        # rank
+        row.append(i + 1)
+        # model display name
+        row.append(model_name)
+
+        row.append(
+            model_table_df["Language Model"].values[model_key]
+        )
+        row.append(
+            model_table_df["Open Source"].values[model_key]
+        )
+        row.append(
+            model_table_df["Regular Text"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["Irregular Text"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["Artistic Text"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["Handwriting"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["Digit string"].values[model_key]
+        )
+
+        row.append(
+            model_table_df["Non-semantic Text"].values[model_key]
+        )
+        row.append(
+            model_table_df["ALL"].values[model_key]
+        )
+        values.append(row)
+    return values
+
+def build_leaderboard_tab(leaderboard_table_file, text_recog_file, Inaccessible_model_file, show_plot=False):
+    if leaderboard_table_file:
+        data = load_leaderboard_table_csv(leaderboard_table_file)
+        data_recog = load_leaderboard_table_csv(text_recog_file)
+        data_Inaccessible = load_leaderboard_table_csv(Inaccessible_model_file)
+        model_table_df = pd.DataFrame(data)
+        model_table_df_Inaccessible = pd.DataFrame(data_Inaccessible)
+        recog_table_df = pd.DataFrame(data_recog)
+        md_head = f"""
+        # 🏆 OCRBench Leaderboard
+        | [GitHub](https://github.com/Yuliang-Liu/MultimodalOCR) | [Paper](https://arxiv.org/abs/2305.07895) |
+        """
+        gr.Markdown(md_head, elem_id="leaderboard_markdown")
+        with gr.Tabs() as tabs:
+            # arena table
+            with gr.Tab("OCRBench", id=0):
+                arena_table_vals = get_arena_table(model_table_df)
+                md = "OCRBench is a comprehensive evaluation benchmark designed to assess the OCR capabilities of Large Multimodal Models. It comprises five components: Text Recognition, SceneText-Centric VQA, Document-Oriented VQA, Key Information Extraction, and Handwritten Mathematical Expression Recognition. The benchmark includes 1000 question-answer pairs, and all the answers undergo manual verification and correction to ensure a more precise evaluation."
+                gr.Markdown(md, elem_id="leaderboard_markdown")
+                gr.Dataframe(
+                    headers=[
+                        "Rank",
+                        "Name",
+                        "Language Model",
+                        "Open Source",
+                        "Text Recognition",
+                        "Scene Text-Centric VQA",
+                        "Doc-Oriented VQA",
+                        "KIE",
+                        "HMER",
+                        "Final Score",
+                    ],
+                    datatype=[
+                        "str",
+                        "markdown",
+                        "str",
+                        "str",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                    ],
+                    value=arena_table_vals,
+                    elem_id="arena_leaderboard_dataframe",
+                    height=700,
+                    column_widths=[60, 120,150,100, 150, 200, 180, 80, 80, 160],
+                    wrap=True,
+                )
+            with gr.Tab("Text Recognition", id=1):
+                arena_table_vals = get_recog_table(recog_table_df)
+                md = "OCRBench is a comprehensive evaluation benchmark designed to assess the OCR capabilities of Large Multimodal Models. It comprises five components: Text Recognition, SceneText-Centric VQA, Document-Oriented VQA, Key Information Extraction, and Handwritten Mathematical Expression Recognition. The benchmark includes 1000 question-answer pairs, and all the answers undergo manual verification and correction to ensure a more precise evaluation."
+                gr.Markdown(md, elem_id="leaderboard_markdown")
+                gr.Dataframe(
+                    headers=[
+                        "Rank",
+                        "Name",
+                        "Language Model",
+                        "Open Source",
+                        "Regular Text",
+                        "Irregular Text",
+                        "Artistic Text",
+                        "Handwriting",
+                        "Digit string",
+                        "Non-semantic Text",
+                        "ALL",
+                    ],
+                    datatype=[
+                        "str",
+                        "markdown",
+                        "str",
+                        "str",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                    ],
+                    value=arena_table_vals,
+                    elem_id="arena_leaderboard_dataframe",
+                    height=700,
+                    column_widths=[60, 120,150,100, 100, 100, 100, 100, 100,100, 80],
+                    wrap=True,
+                )
+            with gr.Tab("Inaccessible Model", id=2):
+                arena_table_vals = get_arena_table(model_table_df_Inaccessible)
+                md = "The models on this list are neither open-source nor have API call interfaces available."
+                gr.Markdown(md, elem_id="leaderboard_markdown")
+                gr.Dataframe(
+                    headers=[
+                        "Rank",
+                        "Name",
+                        "Language Model",
+                        "Open Source",
+                        "Text Recognition",
+                        "Scene Text-Centric VQA",
+                        "Doc-Oriented VQA",
+                        "KIE",
+                        "HMER",
+                        "Final Score",
+                    ],
+                    datatype=[
+                        "str",
+                        "markdown",
+                        "str",
+                        "str",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                    ],
+                    value=arena_table_vals,
+                    elem_id="arena_leaderboard_dataframe",
+                    height=700,
+                    column_widths=[60, 120,150,100, 150, 200, 180, 80, 80, 160],
+                    wrap=True,
+                )
+    else:
+        pass
+    md_tail = f"""
+    # Notice
+    Sometimes, API calls to closed-source models may not succeed. In such cases, we will repeat the calls for unsuccessful samples until it becomes impossible to obtain a successful response. It is important to note that due to rigorous security reviews by OpenAI, GPT4V refuses to provide results for the 84 samples in OCRBench.
+    If you would like to include your model in the OCRBench leaderboard, please follow the evaluation instructions provided on [GitHub](https://github.com/Yuliang-Liu/MultimodalOCR), [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) or [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval) and feel free to contact us via email at [email protected]. We will update the leaderboard in time."""
+    gr.Markdown(md_tail, elem_id="leaderboard_markdown")
+
+def build_demo(leaderboard_table_file, recog_table_file, Inaccessible_model_file):
+    text_size = gr.themes.sizes.text_lg
+
+    with gr.Blocks(
+        title="OCRBench Leaderboard",
+        theme=gr.themes.Base(text_size=text_size),
+        css=block_css,
+    ) as demo:
+        leader_components = build_leaderboard_tab(
+            leaderboard_table_file, recog_table_file,Inaccessible_model_file,show_plot=True
+        )
+    return demo
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--share", action="store_true")
+    parser.add_argument("--OCRBench_file", type=str, default="./OCRBench.csv")
+    parser.add_argument("--TextRecognition_file", type=str, default="./TextRecognition.csv")
+    parser.add_argument("--Inaccessible_model_file", type=str, default="./Inaccessible_model.csv")
+    args = parser.parse_args()
+
+    demo = build_demo(args.OCRBench_file, args.TextRecognition_file, args.Inaccessible_model_file)
+    demo.launch()

gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text