First entries and streamlit app

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "takumii.markdowntable"
+    ]
+}

README.md

@@ -1,10 +1,18 @@
 # llm-leaderboard
 A joint community effort to create one central leaderboard for LLMs
 
+Visit the interactive leaderboard at TODO.
+
 ### Leaderboard
 
-| a | b |
-|---|---|
-| 1 | 4 |
-| 2 | 5 |
-| 3 | 6 |
+| Model Name                                                                                                            | [Chatbot Arena Elo](https://lmsys.org/blog/2023-05-03-arena/) |
+| --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- |
+| [alpaca-13b](https://crfm.stanford.edu/2023/03/13/alpaca.html)                                                        | 1008                                                          |
+| [chatglm-6b](https://chatglm.cn/blog)                                                                                 | 985                                                           |
+| [dolly-v2-12b](https://www.databricks.com/blog/2023/04/12/dolly-first-open-commercially-viable-instruction-tuned-llm) | 944                                                           |
+| [fastchat-t5-3b](https://huggingface.co/lmsys/fastchat-t5-3b-v1.0)                                                    | 951                                                           |
+| [koala-13b](https://bair.berkeley.edu/blog/2023/04/03/koala/)                                                         | 1082                                                          |
+| [llama-13b](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/)                                         | 932                                                           |
+| [stablelm-tuned-alpha-7b](https://github.com/stability-AI/stableLM)                                                   | 858                                                           |
+| [vicuna-13b](https://lmsys.org/blog/2023-03-30-vicuna/)                                                               | 1169                                                          |
+| [oasst-pythia-12b](https://open-assistant.io/)                                                                        | 1065                                                          |

requirements-dev.txt

@@ -0,0 +1,3 @@
+black
+flake
+mypy

requirements.txt

@@ -0,0 +1,2 @@
+pandas~=2.0.1
+streamlit~=1.22.0

streamlit_app.py

@@ -0,0 +1,100 @@
+import pandas as pd
+import streamlit as st
+import io
+import requests
+
+REPO_URL = "https://github.com/LudwigStumpp/llm-leaderboard"
+
+
+def grab_readme_file_from_repo(repo_url: str) -> str:
+    """Grabs the README.md file from a GitHub repository.
+
+    Args:
+        repo_url (str): URL of the GitHub repository.
+
+    Returns:
+        str: Content of the README.md file.
+    """
+    readme_url = repo_url.replace("github.com", "raw.githubusercontent.com") + "/main/README.md"
+    readme = requests.get(readme_url).text
+    return readme
+
+
+def extract_markdown_table_from_multiline(multiline: str, table_headline: str) -> str:
+    """Extracts the markdown table from a multiline string.
+
+    Args:
+        multiline (str): content of README.md file.
+        table_headline (str): Headline of the table in the README.md file.
+
+    Returns:
+        str: Markdown table.
+
+    Raises:
+        ValueError: If the table could not be found.
+    """
+    # extract everything between the table headline and the next headline
+    table = []
+    start = False
+    for line in multiline.split("\n"):
+        if line.startswith(table_headline):
+            start = True
+        elif line.startswith("###"):
+            start = False
+        elif start:
+            table.append(line + "\n")
+
+    if len(table) == 0:
+        raise ValueError(f"Could not find table with headline '{table_headline}'")
+
+    return "".join(table)
+
+
+def setup_basic():
+    title = "LLM-Leaderboard"
+
+    st.set_page_config(
+        page_title=title,
+        page_icon="🏆",
+    )
+    st.title(title)
+
+    st.markdown(
+        """
+        A joint community effort to create one central leaderboard for LLMs.
+        Visit [llm-leaderboard](https://github.com/LudwigStumpp/llm-leaderboard) to contribute.
+        """
+    )
+
+
+def setup_table():
+    readme = grab_readme_file_from_repo(REPO_URL)
+    markdown_table = extract_markdown_table_from_multiline(readme, table_headline="### Leaderboard")
+
+    df = (
+        pd.read_table(io.StringIO(markdown_table), sep="|", header=0, skipinitialspace=True, index_col=1)
+        .dropna(axis=1, how="all")  # drop empty columns
+        .iloc[1:]  # drop first row which is the "----" separator of the original markdown table
+    )
+
+    # show interactive table
+    st.dataframe(df)
+
+
+def setup_footer():
+    st.markdown(
+        """
+        ---
+        Made with ❤️ by the awesome open-source community from all over 🌍.
+        """
+    )
+
+
+def main():
+    setup_basic()
+    setup_table()
+    setup_footer()
+
+
+if __name__ == "__main__":
+    main()