+ {`from transformers import AutoConfig, AutoModel, AutoTokenizer
+
+config = AutoConfig.from_pretrained("your-username/your-model", revision="main")
+model = AutoModel.from_pretrained("your-username/your-model", revision="main")
+tokenizer = AutoTokenizer.from_pretrained("your-username/your-model", revision="main")`}
+
+ {citation}
+ {error}
" - - -def styled_warning(warn): - return f"{warn}
" - - -def styled_message(message): - return f"{message}
" - - -def has_no_nan_values(df, columns): - return df[columns].notna().all(axis=1) - - -def has_nan_values(df, columns): - return df[columns].isna().any(axis=1) diff --git a/src/display/utils.py b/src/display/utils.py deleted file mode 100644 index aad01c0cd1c4f23d463ed73face884eae054bbe4..0000000000000000000000000000000000000000 --- a/src/display/utils.py +++ /dev/null @@ -1,272 +0,0 @@ -from dataclasses import dataclass, make_dataclass -from datasets import load_dataset -from enum import Enum -import json -import logging -from datetime import datetime -import pandas as pd - -from src.envs import OFFICIAL_PROVIDERS_REPO - -# Configure logging -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") - -dataset = load_dataset(OFFICIAL_PROVIDERS_REPO) -official_providers = dataset["train"][0]["CURATED_SET"] - -# Convert ISO 8601 dates to datetime objects for comparison -def parse_iso8601_datetime(date_str): - if date_str.endswith('Z'): - date_str = date_str[:-1] + '+00:00' - return datetime.fromisoformat(date_str) - -def parse_datetime(datetime_str): - formats = [ - "%Y-%m-%dT%H-%M-%S.%f", # Format with dashes - "%Y-%m-%dT%H:%M:%S.%f", # Standard format with colons - "%Y-%m-%dT%H %M %S.%f", # Spaces as separator - ] - - for fmt in formats: - try: - return datetime.strptime(datetime_str, fmt) - except ValueError: - continue - # in rare cases set unix start time for files with incorrect time (legacy files) - logging.error(f"No valid date format found for: {datetime_str}") - return datetime(1970, 1, 1) - - -def load_json_data(file_path): - """Safely load JSON data from a file.""" - try: - with open(file_path, "r") as file: - return json.load(file) - except json.JSONDecodeError: - print(f"Error reading JSON from {file_path}") - return None # Or raise an exception - - -def fields(raw_class): - return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"] - - -@dataclass -class Task: - benchmark: str - metric: str - col_name: str - - -class Tasks(Enum): - ifeval = Task("leaderboard_ifeval", "strict_acc,none", "IFEval") - ifeval_raw = Task("leaderboard_ifeval", "strict_acc,none", "IFEval Raw") - - bbh = Task("leaderboard_bbh", "acc_norm,none", "BBH") - bbh_raw = Task("leaderboard_bbh", "acc_norm,none", "BBH Raw") - - math = Task("leaderboard_math_hard", "exact_match,none", "MATH Lvl 5") - math_raw = Task("leaderboard_math_hard", "exact_match,none", "MATH Lvl 5 Raw") - - gpqa = Task("leaderboard_gpqa", "acc_norm,none", "GPQA") - gpqa_raw = Task("leaderboard_gpqa", "acc_norm,none", "GPQA Raw") - - musr = Task("leaderboard_musr", "acc_norm,none", "MUSR") - musr_raw = Task("leaderboard_musr", "acc_norm,none", "MUSR Raw") - - mmlu_pro = Task("leaderboard_mmlu_pro", "acc,none", "MMLU-PRO") - mmlu_pro_raw = Task("leaderboard_mmlu_pro", "acc,none", "MMLU-PRO Raw") - - -# These classes are for user facing column names, -# to avoid having to change them all around the code -# when a modif is needed -@dataclass(frozen=True) -class ColumnContent: - name: str - type: str - displayed_by_default: bool - hidden: bool = False - never_hidden: bool = False - dummy: bool = False - - -auto_eval_column_dict = [] -# Init -auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)]) -auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)]) -# Scores -auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]) -for task in Tasks: - displayed_by_default = not task.name.endswith("_raw") - auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", displayed_by_default=displayed_by_default)]) -# Model information -auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)]) -auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)]) -auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)]) -auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)]) -auto_eval_column_dict.append(["merged", ColumnContent, ColumnContent("Merged", "bool", False)]) -auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)]) -auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)]) -auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)]) -auto_eval_column_dict.append( - ["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, hidden=True)] -) -auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]) -auto_eval_column_dict.append(["not_flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)]) -auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)]) - -auto_eval_column_dict.append(["submission_date", ColumnContent, ColumnContent("Submission Date", "bool", False, hidden=False)]) -auto_eval_column_dict.append(["upload_to_hub", ColumnContent, ColumnContent("Upload To Hub Date", "bool", False, hidden=False)]) - -auto_eval_column_dict.append(["use_chat_template", ColumnContent, ColumnContent("Chat Template", "bool", False)]) -auto_eval_column_dict.append(["official_providers", ColumnContent, ColumnContent("Official Providers", "bool", False, hidden=True)]) - -# fullname structure:Baseline
", -# AutoEvalColumn.revision.name: "N/A", -# AutoEvalColumn.precision.name: None, -# AutoEvalColumn.merged.name: False, -# AutoEvalColumn.average.name: 31.0, -# AutoEvalColumn.arc.name: 25.0, -# AutoEvalColumn.hellaswag.name: 25.0, -# AutoEvalColumn.mmlu.name: 25.0, -# AutoEvalColumn.truthfulqa.name: 25.0, -# AutoEvalColumn.winogrande.name: 50.0, -# AutoEvalColumn.gsm8k.name: 0.21, -# AutoEvalColumn.fullname.name: "baseline", -# AutoEvalColumn.model_type.name: "", -# AutoEvalColumn.not_flagged.name: False, -# } - -# Average ⬆️ human baseline is 0.897 (source: averaging human baselines below) -# ARC human baseline is 0.80 (source: https://lab42.global/arc/) -# HellaSwag human baseline is 0.95 (source: https://deepgram.com/learn/hellaswag-llm-benchmark-guide) -# MMLU human baseline is 0.898 (source: https://openreview.net/forum?id=d7KBjmI3GmQ) -# TruthfulQA human baseline is 0.94(source: https://arxiv.org/pdf/2109.07958.pdf) -# Winogrande: https://leaderboard.allenai.org/winogrande/submissions/public -# GSM8K: paper -# Define the human baselines -# human_baseline_row = { -# AutoEvalColumn.model.name: "Human performance
", -# AutoEvalColumn.revision.name: "N/A", -# AutoEvalColumn.precision.name: None, -# AutoEvalColumn.average.name: 92.75, -# AutoEvalColumn.merged.name: False, -# AutoEvalColumn.arc.name: 80.0, -# AutoEvalColumn.hellaswag.name: 95.0, -# AutoEvalColumn.mmlu.name: 89.8, -# AutoEvalColumn.truthfulqa.name: 94.0, -# AutoEvalColumn.winogrande.name: 94.0, -# AutoEvalColumn.gsm8k.name: 100, -# AutoEvalColumn.fullname.name: "human_baseline", -# AutoEvalColumn.model_type.name: "", -# AutoEvalColumn.not_flagged.name: False, -# } - - -@dataclass -class ModelDetails: - name: str - symbol: str = "" # emoji, only for the model type - - -class ModelType(Enum): - PT = ModelDetails(name="🟢 pretrained", symbol="🟢") - CPT = ModelDetails(name="🟩 continuously pretrained", symbol="🟩") - FT = ModelDetails(name="🔶 fine-tuned on domain-specific datasets", symbol="🔶") - MM = ModelDetails(name="🌸 multimodal", symbol="🌸") - chat = ModelDetails(name="💬 chat models (RLHF, DPO, IFT, ...)", symbol="💬") - merges = ModelDetails(name="🤝 base merges and moerges", symbol="🤝") - Unknown = ModelDetails(name="❓ other", symbol="❓") - - def to_str(self, separator=" "): - return f"{self.value.symbol}{separator}{self.value.name}" - - @staticmethod - def from_str(m_type): - if any([k for k in m_type if k in ["fine-tuned","🔶", "finetuned"]]): - return ModelType.FT - if "continuously pretrained" in m_type or "🟩" in m_type: - return ModelType.CPT - if "pretrained" in m_type or "🟢" in m_type: - return ModelType.PT - if any([k in m_type for k in ["instruction-tuned", "RL-tuned", "chat", "🟦", "⭕", "💬"]]): - return ModelType.chat - if "merge" in m_type or "🤝" in m_type: - return ModelType.merges - if "multimodal" in m_type or "🌸" in m_type: - return ModelType.MM - return ModelType.Unknown - -class WeightType(Enum): - Adapter = ModelDetails("Adapter") - Original = ModelDetails("Original") - Delta = ModelDetails("Delta") - - -class Precision(Enum): - float16 = ModelDetails("float16") - bfloat16 = ModelDetails("bfloat16") - qt_8bit = ModelDetails("8bit") - qt_4bit = ModelDetails("4bit") - qt_GPTQ = ModelDetails("GPTQ") - Unknown = ModelDetails("?") - - @staticmethod - def from_str(precision): - if precision in ["torch.float16", "float16"]: - return Precision.float16 - if precision in ["torch.bfloat16", "bfloat16"]: - return Precision.bfloat16 - if precision in ["8bit"]: - return Precision.qt_8bit - if precision in ["4bit"]: - return Precision.qt_4bit - if precision in ["GPTQ", "None"]: - return Precision.qt_GPTQ - return Precision.Unknown - - -# Column selection -COLS = [c.name for c in fields(AutoEvalColumn)] -TYPES = [c.type for c in fields(AutoEvalColumn)] - -EVAL_COLS = [c.name for c in fields(EvalQueueColumn)] -EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)] - -BENCHMARK_COLS = [t.value.col_name for t in Tasks] - -NUMERIC_INTERVALS = { - "?": pd.Interval(-1, 0, closed="right"), - "~1.5": pd.Interval(0, 2, closed="right"), - "~3": pd.Interval(2, 4, closed="right"), - "~7": pd.Interval(4, 9, closed="right"), - "~13": pd.Interval(9, 20, closed="right"), - "~35": pd.Interval(20, 45, closed="right"), - "~60": pd.Interval(45, 70, closed="right"), - "70+": pd.Interval(70, 10000, closed="right"), -} diff --git a/src/envs.py b/src/envs.py deleted file mode 100644 index fd4c52f52630201c54c2641ea2fc4a5f468c731b..0000000000000000000000000000000000000000 --- a/src/envs.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -from huggingface_hub import HfApi - -# clone / pull the lmeh eval data -HF_TOKEN = os.environ.get("HF_TOKEN", None) - -REPO_ID = "open-llm-leaderboard/open_llm_leaderboard" -QUEUE_REPO = "open-llm-leaderboard/requests" -AGGREGATED_REPO = "open-llm-leaderboard/contents" -VOTES_REPO = "open-llm-leaderboard/votes" -OFFICIAL_PROVIDERS_REPO = "open-llm-leaderboard/official-providers" - -HF_HOME = os.getenv("HF_HOME", ".") - -# Check HF_HOME write access -print(f"Initial HF_HOME set to: {HF_HOME}") - -if not os.access(HF_HOME, os.W_OK): - print(f"No write access to HF_HOME: {HF_HOME}. Resetting to current directory.") - HF_HOME = "." - os.environ["HF_HOME"] = HF_HOME -else: - print("Write access confirmed for HF_HOME") - -VOTES_PATH = os.path.join(HF_HOME, "model-votes") -EVAL_REQUESTS_PATH = os.path.join(HF_HOME, "eval-queue") - -# Rate limit variables -RATE_LIMIT_PERIOD = 7 -RATE_LIMIT_QUOTA = 5 -HAS_HIGHER_RATE_LIMIT = [] - -API = HfApi(token=HF_TOKEN) diff --git a/src/leaderboard/filter_models.py b/src/leaderboard/filter_models.py deleted file mode 100644 index 323384465c3c617402b6ffb7471c4ada4a84b813..0000000000000000000000000000000000000000 --- a/src/leaderboard/filter_models.py +++ /dev/null @@ -1,75 +0,0 @@ -from src.display.formatting import model_hyperlink -from src.display.utils import AutoEvalColumn - - -# Models which have been flagged by users as being problematic for a reason or another -# (Model name to forum discussion link) -# None for the v2 so far! -FLAGGED_MODELS = {} - -# Models which have been requested by orgs to not be submitted on the leaderboard -DO_NOT_SUBMIT_MODELS = [ - "Voicelab/trurl-2-13b", # trained on MMLU - "TigerResearch/tigerbot-70b-chat", # per authors request - "TigerResearch/tigerbot-70b-chat-v2", # per authors request - "TigerResearch/tigerbot-70b-chat-v4-4k", # per authors request -] - - -def flag_models(leaderboard_data: list[dict]): - """Flags models based on external criteria or flagged status.""" - for model_data in leaderboard_data: - # Skip flagging if official providers is True - if model_data.get(AutoEvalColumn.official_providers.name, False): - model_data[AutoEvalColumn.not_flagged.name] = True - continue - - # If a model is not flagged, use its "fullname" as a key - if model_data[AutoEvalColumn.not_flagged.name]: - flag_key = model_data[AutoEvalColumn.fullname.name] - else: - flag_key = None - - # Reverse the logic: Check for non-flagged models instead - if flag_key in FLAGGED_MODELS: - issue_num = FLAGGED_MODELS[flag_key].split("/")[-1] - issue_link = model_hyperlink( - FLAGGED_MODELS[flag_key], - f"See discussion #{issue_num}", - ) - model_data[AutoEvalColumn.model.name] = ( - f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}" - ) - model_data[AutoEvalColumn.not_flagged.name] = False - else: - model_data[AutoEvalColumn.not_flagged.name] = True - - -def remove_forbidden_models(leaderboard_data: list[dict]): - """Removes models from the leaderboard based on the DO_NOT_SUBMIT list.""" - indices_to_remove = [] - for ix, model in enumerate(leaderboard_data): - if model[AutoEvalColumn.fullname.name] in DO_NOT_SUBMIT_MODELS: - indices_to_remove.append(ix) - - # Remove the models from the list - for ix in reversed(indices_to_remove): - leaderboard_data.pop(ix) - return leaderboard_data - -""" -def remove_forbidden_models(leaderboard_data): - #Removes models from the leaderboard based on the DO_NOT_SUBMIT list. - indices_to_remove = [] - for ix, row in leaderboard_data.iterrows(): - if row[AutoEvalColumn.fullname.name] in DO_NOT_SUBMIT_MODELS: - indices_to_remove.append(ix) - - # Remove the models from the list - return leaderboard_data.drop(indices_to_remove) -""" - - -def filter_models_flags(leaderboard_data: list[dict]): - leaderboard_data = remove_forbidden_models(leaderboard_data) - flag_models(leaderboard_data) diff --git a/src/pages/LeaderboardPage/components/Leaderboard/components/ColumnSelector/ColumnSelector.js b/src/pages/LeaderboardPage/components/Leaderboard/components/ColumnSelector/ColumnSelector.js new file mode 100644 index 0000000000000000000000000000000000000000..f6ba4b163598d5129ea30950b22706a62412f988 --- /dev/null +++ b/src/pages/LeaderboardPage/components/Leaderboard/components/ColumnSelector/ColumnSelector.js @@ -0,0 +1,2 @@ +import FilterTag from "../../../../../../components/shared/FilterTag"; +import InfoIconWithTooltip from "../../../../../../components/shared/InfoIconWithTooltip"; diff --git a/src/populate.py b/src/populate.py deleted file mode 100644 index 41e5f8ce6ff2782071696d1b53648008584a5a65..0000000000000000000000000000000000000000 --- a/src/populate.py +++ /dev/null @@ -1,54 +0,0 @@ -import pathlib -import pandas as pd -from datasets import Dataset -from src.display.formatting import has_no_nan_values, make_clickable_model -from src.display.utils import AutoEvalColumn, EvalQueueColumn -from src.leaderboard.filter_models import filter_models_flags -from src.display.utils import load_json_data - - -def _process_model_data(entry, model_name_key="model", revision_key="revision"): - """Enrich model data with clickable links and revisions.""" - entry[EvalQueueColumn.model_name.name] = entry.get(model_name_key, "") - entry[EvalQueueColumn.model_link.name] = make_clickable_model(entry.get(model_name_key, "")) - entry[EvalQueueColumn.revision.name] = entry.get(revision_key, "main") - return entry - - -def get_evaluation_queue_df(save_path, cols): - """Generate dataframes for pending, running, and finished evaluation entries.""" - save_path = pathlib.Path(save_path) - all_evals = [] - - for path in save_path.rglob("*.json"): - data = load_json_data(path) - if data: - all_evals.append(_process_model_data(data)) - - # Organizing data by status - status_map = { - "PENDING": ["PENDING", "RERUN"], - "RUNNING": ["RUNNING"], - "FINISHED": ["FINISHED", "PENDING_NEW_EVAL"], - } - status_dfs = {status: [] for status in status_map} - for eval_data in all_evals: - for status, extra_statuses in status_map.items(): - if eval_data["status"] in extra_statuses: - status_dfs[status].append(eval_data) - - return tuple(pd.DataFrame(status_dfs[status], columns=cols) for status in ["FINISHED", "RUNNING", "PENDING"]) - - -def get_leaderboard_df(leaderboard_dataset: Dataset, cols: list, benchmark_cols: list): - """Retrieve and process leaderboard data.""" - all_data_json = leaderboard_dataset.to_dict() - num_items = leaderboard_dataset.num_rows - all_data_json_list = [{k: all_data_json[k][ix] for k in all_data_json.keys()} for ix in range(num_items)] - filter_models_flags(all_data_json_list) - - df = pd.DataFrame.from_records(all_data_json_list) - df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False) - df = df[cols].round(decimals=2) - df = df[has_no_nan_values(df, benchmark_cols)] - return df \ No newline at end of file diff --git a/src/submission/check_validity.py b/src/submission/check_validity.py deleted file mode 100644 index 2d952de9aa6c14ed179f95ae29f45f90bbf3babb..0000000000000000000000000000000000000000 --- a/src/submission/check_validity.py +++ /dev/null @@ -1,230 +0,0 @@ -import json -import os -import re -import logging -from collections import defaultdict -from datetime import datetime, timedelta, timezone - -import huggingface_hub -from huggingface_hub import ModelCard, hf_hub_download -from huggingface_hub.hf_api import ModelInfo, get_safetensors_metadata, parse_safetensors_file_metadata -from transformers import AutoConfig, AutoTokenizer - -from src.display.utils import parse_iso8601_datetime, official_providers -from src.envs import HAS_HIGHER_RATE_LIMIT - - -# ht to @Wauplin, thank you for the snippet! -# See https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard/discussions/317 -def check_model_card(repo_id: str) -> tuple[bool, str]: - # Returns operation status, and error message - try: - card = ModelCard.load(repo_id) - except huggingface_hub.utils.EntryNotFoundError: - return False, "Please add a model card to your model to explain how you trained/fine-tuned it.", None - - # Enforce license metadata - if card.data.license is None and not ("license_name" in card.data and "license_link" in card.data): - return ( - False, - ( - "License not found. Please add a license to your model card using the `license` metadata or a" - " `license_name`/`license_link` pair." - ), - None, - ) - - # Enforce card content - if len(card.text) < 200: - return False, "Please add a description to your model card, it is too short.", None - - return True, "", card - - -def is_model_on_hub( - model_name: str, revision: str, token: str | None = None, trust_remote_code: bool = False, test_tokenizer: bool = False, -) -> tuple[bool, str, AutoConfig]: - try: - config = AutoConfig.from_pretrained( - model_name, revision=revision, trust_remote_code=trust_remote_code, token=token, force_download=True) - if test_tokenizer: - try: - AutoTokenizer.from_pretrained( - model_name, revision=revision, trust_remote_code=trust_remote_code, token=token, - ) - except ValueError as e: - return (False, f"uses a tokenizer which is not in a transformers release: {e}", None) - except Exception: - return ( - False, - "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", - None, - ) - except Exception: - return ( - False, - "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", - None, - ) - return True, None, config - - except ValueError: - return ( - False, - "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.", - None, - ) - - except Exception as e: - if "You are trying to access a gated repo." in str(e): - return True, "uses a gated model.", None - return False, f"was not found or misconfigured on the hub! Error raised was {e.args[0]}", None - - -def get_model_size(model_info: ModelInfo, precision: str, base_model: str| None) -> tuple[float | None, str]: - size_pattern = re.compile(r"(\d+\.)?\d+(b|m)") - safetensors = None - adapter_safetensors = None - # hack way to check that model is adapter - is_adapter = "adapter_config.json" in (s.rfilename for s in model_info.siblings) - - try: - if is_adapter: - if not base_model: - return None, "Adapter model submission detected. Please ensure the base model information is provided." - - adapter_safetensors = parse_safetensors_file_metadata(model_info.id, "adapter_model.safetensors") - safetensors = get_safetensors_metadata(base_model) - else: - safetensors = get_safetensors_metadata(model_info.id) - except Exception as e: - logging.warning(f"Failed to get safetensors metadata for model {model_info.id}: {e!s}") - - if safetensors is not None: - model_size = sum(safetensors.parameter_count.values()) - if adapter_safetensors is not None: - model_size += sum(safetensors.parameter_count.values()) - model_size = round(model_size / 1e9, 3) - else: - try: - size_match = re.search(size_pattern, model_info.id.lower()) - if size_match: - model_size = size_match.group(0) - model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3) - else: - return None, "Unknown model size" - except AttributeError: - logging.warning(f"Unable to parse model size from ID: {model_info.id}") - return None, "Unknown model size" - - size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.id.lower()) else 1 - model_size = size_factor * model_size - - return model_size, "" - -def get_model_arch(model_info: ModelInfo): - return model_info.config.get("architectures", "Unknown") - - -def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota): - # No limit for official providers - if org_or_user in official_providers: - return True, "" - # Increase quota first if user has higher limits - if org_or_user in HAS_HIGHER_RATE_LIMIT: - rate_limit_quota *= 2 - - if org_or_user not in users_to_submission_dates: - return True, "" - - submission_dates = sorted(users_to_submission_dates[org_or_user]) - time_limit = datetime.now(timezone.utc) - timedelta(days=rate_limit_period) - - submissions_after_timelimit = [ - parse_iso8601_datetime(d) for d in submission_dates - if parse_iso8601_datetime(d) > time_limit - ] - - num_models_submitted_in_period = len(submissions_after_timelimit) - - # Use >= to correctly enforce the rate limit - if num_models_submitted_in_period >= rate_limit_quota: - error_msg = f"Organisation or user `{org_or_user}` already has {num_models_submitted_in_period} model requests submitted in the last {rate_limit_period} days.\n" - error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗" - return False, error_msg - - return True, "" - - -def already_submitted_models(requested_models_dir: str) -> set[str]: - depth = 1 - file_names = [] - users_to_submission_dates = defaultdict(list) - - for root, _, files in os.walk(requested_models_dir): - current_depth = root.count(os.sep) - requested_models_dir.count(os.sep) - if current_depth == depth: - for file in files: - if not file.endswith(".json"): - continue - with open(os.path.join(root, file), "r") as f: - info = json.load(f) - file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}") - - # Select organisation - if info["model"].count("/") == 0 or "submitted_time" not in info: - continue - organisation, _ = info["model"].split("/") - users_to_submission_dates[organisation].append(info["submitted_time"]) - - return set(file_names), users_to_submission_dates - -def check_chat_template(model: str, revision: str) -> tuple[bool, str]: - try: - # Attempt to download only the tokenizer_config.json file - config_file = hf_hub_download( - repo_id=model, - filename="tokenizer_config.json", - revision=revision, - repo_type="model" - ) - - # Read and parse the tokenizer_config.json file - with open(config_file, 'r') as f: - tokenizer_config = json.load(f) - - # Check if chat_template exists in the tokenizer configuration - if 'chat_template' not in tokenizer_config: - return False, f"The model {model} doesn't have a chat_template in its tokenizer_config.json. Please add a chat_template before submitting or submit without it." - - return True, "" - except Exception as e: - return False, f"Error checking chat_template for model {model}: {str(e)}" - -def get_model_tags(model_card, model: str): - is_merge_from_metadata = False - is_moe_from_metadata = False - - tags = [] - if model_card is None: - return tags - if model_card.data.tags: - is_merge_from_metadata = any( - [tag in model_card.data.tags for tag in ["merge", "moerge", "mergekit", "lazymergekit"]] - ) - is_moe_from_metadata = any([tag in model_card.data.tags for tag in ["moe", "moerge"]]) - - is_merge_from_model_card = any( - keyword in model_card.text.lower() for keyword in ["merged model", "merge model", "moerge"] - ) - if is_merge_from_model_card or is_merge_from_metadata: - tags.append("merge") - is_moe_from_model_card = any(keyword in model_card.text.lower() for keyword in ["moe", "mixtral"]) - # Hardcoding because of gating problem - if "Qwen/Qwen1.5-32B" in model: - is_moe_from_model_card = False - is_moe_from_name = "moe" in model.lower().replace("/", "-").replace("_", "-").split("-") - if is_moe_from_model_card or is_moe_from_name or is_moe_from_metadata: - tags.append("moe") - - return tags diff --git a/src/submission/submit.py b/src/submission/submit.py deleted file mode 100644 index fcb3f63237237e02772b59245682b9fb27a6d813..0000000000000000000000000000000000000000 --- a/src/submission/submit.py +++ /dev/null @@ -1,228 +0,0 @@ -import json -import os -import gradio as gr -from datetime import datetime, timezone - -from dataclasses import dataclass -from transformers import AutoConfig - -from src.display.formatting import styled_error, styled_message, styled_warning -from src.envs import ( - API, - EVAL_REQUESTS_PATH, - HF_TOKEN, - QUEUE_REPO, - RATE_LIMIT_PERIOD, - RATE_LIMIT_QUOTA, - VOTES_REPO, - VOTES_PATH, -) -from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS -from src.submission.check_validity import ( - already_submitted_models, - check_model_card, - get_model_size, - is_model_on_hub, - user_submission_permission, - check_chat_template, -) - -from src.voting.vote_system import VoteManager - -REQUESTED_MODELS = None -USERS_TO_SUBMISSION_DATES = None - -vote_manager = VoteManager(VOTES_PATH, EVAL_REQUESTS_PATH, VOTES_REPO) - -@dataclass -class ModelSizeChecker: - model: str - precision: str - model_size_in_b: float - - def get_precision_factor(self): - if self.precision in ["float16", "bfloat16"]: - return 1 - elif self.precision == "8bit": - return 2 - elif self.precision == "4bit": - return 4 - elif self.precision == "GPTQ": - config = AutoConfig.from_pretrained(self.model) - num_bits = int(config.quantization_config["bits"]) - bits_to_precision_factor = {2: 8, 3: 6, 4: 4, 8: 2} - return bits_to_precision_factor.get(num_bits, 1) - else: - raise Exception(f"Unknown precision {self.precision}.") - - def can_evaluate(self): - precision_factor = self.get_precision_factor() - return self.model_size_in_b <= 140 * precision_factor - -def add_new_eval( - model: str, - base_model: str, - revision: str, - precision: str, - weight_type: str, - model_type: str, - use_chat_template: bool, - profile: gr.OAuthProfile | None, - requested_models: set[str] = None, - users_to_submission_dates: dict[str, list[str]] = None, -): - # Login required - if profile is None: - return styled_error("Hub Login Required") - - # Name of the actual user who sent the request - username = profile.username - - # Initialize the requested_models and users_to_submission_dates variables - # If the caller did not provide these values, fetch them from the EVAL_REQUESTS_PATH - if requested_models is None or users_to_submission_dates is None: - requested_models, users_to_submission_dates = already_submitted_models(EVAL_REQUESTS_PATH) - - org_or_user = "" - model_path = model - if "/" in model: - org_or_user = model.split("/")[0] - model_path = model.split("/")[1] - - precision = precision.split(" ")[0] - current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - if model_type is None or model_type == "": - return styled_error("Please select a model type.") - - # Is the user rate limited? - if org_or_user != "": - user_can_submit, error_msg = user_submission_permission( - org_or_user, users_to_submission_dates, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA - ) - if not user_can_submit: - return styled_error(error_msg) - - # Did the model authors forbid its submission to the leaderboard? - if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS: - return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.") - - # Does the model actually exist? - if revision == "": - revision = "main" - try: - model_info = API.model_info(repo_id=model, revision=revision) - except Exception as e: - return styled_error("Could not get your model information. Please fill it up properly.") - - # Has it been submitted already? - model_key = f"{model}_{model_info.sha}_{precision}" - if model_key in requested_models: - return styled_error(f"The model '{model}' with revision '{model_info.sha}' and precision '{precision}' has already been submitted.") - - # Check model size early - model_size, error_text = get_model_size(model_info=model_info, precision=precision, base_model=base_model) - if model_size is None: - return styled_error(error_text) - - # Absolute size limit for float16 and bfloat16 - if precision in ["float16", "bfloat16"] and model_size > 100: - return styled_error(f"Sadly, models larger than 100B parameters cannot be submitted in {precision} precision at this time. " - f"Your model size: {model_size:.2f}B parameters.") - - # Precision-adjusted size limit for 8bit, 4bit, and GPTQ - if precision in ["8bit", "4bit", "GPTQ"]: - size_checker = ModelSizeChecker(model=model, precision=precision, model_size_in_b=model_size) - - if not size_checker.can_evaluate(): - precision_factor = size_checker.get_precision_factor() - max_size = 140 * precision_factor - return styled_error(f"Sadly, models this big ({model_size:.2f}B parameters) cannot be evaluated automatically " - f"at the moment on our cluster. The maximum size for {precision} precision is {max_size:.2f}B parameters.") - - architecture = "?" - # Is the model on the hub? - if weight_type in ["Delta", "Adapter"]: - base_model_on_hub, error, _ = is_model_on_hub( - model_name=base_model, revision="main", token=HF_TOKEN, test_tokenizer=True - ) - if not base_model_on_hub: - return styled_error(f'Base model "{base_model}" {error}') - if not weight_type == "Adapter": - model_on_hub, error, model_config = is_model_on_hub(model_name=model, revision=model_info.sha, test_tokenizer=True) - if not model_on_hub or model_config is None: - return styled_error(f'Model "{model}" {error}') - if model_config is not None: - architectures = getattr(model_config, "architectures", None) - if architectures: - architecture = ";".join(architectures) - # Were the model card and license filled? - try: - model_info.cardData["license"] - except Exception: - return styled_error("Please select a license for your model") - - modelcard_OK, error_msg, model_card = check_model_card(model) - if not modelcard_OK: - return styled_error(error_msg) - - # Check the chat template submission - if use_chat_template: - chat_template_valid, chat_template_error = check_chat_template(model, revision) - if not chat_template_valid: - return styled_error(chat_template_error) - - # Seems good, creating the eval - print("Adding new eval") - eval_entry = { - "model": model, - "base_model": base_model, - "revision": model_info.sha, # force to use the exact model commit - "precision": precision, - "params": model_size, - "architectures": architecture, - "weight_type": weight_type, - "status": "PENDING", - "submitted_time": current_time, - "model_type": model_type, - "job_id": -1, - "job_start_time": None, - "use_chat_template": use_chat_template, - "sender": username - } - - print("Creating eval file") - OUT_DIR = f"{EVAL_REQUESTS_PATH}/{org_or_user}" - os.makedirs(OUT_DIR, exist_ok=True) - out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json" - - with open(out_path, "w") as f: - f.write(json.dumps(eval_entry)) - - print("Uploading eval file") - print(eval_entry) - API.upload_file( - path_or_fileobj=out_path, - path_in_repo=out_path.split("eval-queue/")[1], - repo_id=QUEUE_REPO, - repo_type="dataset", - commit_message=f"Add {model} to eval queue", - ) - - # Remove the local file - os.remove(out_path) - - # Always add a vote for the submitted model - vote_manager.add_vote( - selected_model=model, - pending_models_df=None, - profile=profile - ) - print(f"Automatically added a vote for {model} submitted by {username}") - - # Upload votes to the repository - vote_manager.upload_votes() - - return styled_message( - "Your request and vote has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list." - ) \ No newline at end of file diff --git a/src/tools/create_request_file.py b/src/tools/create_request_file.py deleted file mode 100644 index c1aea64f3fdd25fa5ea087cf02709359c5b5892d..0000000000000000000000000000000000000000 --- a/src/tools/create_request_file.py +++ /dev/null @@ -1,92 +0,0 @@ -import json -import os -import pprint -from datetime import datetime, timezone - -import click -from colorama import Fore -from huggingface_hub import HfApi, snapshot_download - -from src.display.utils import ModelType, WeightType -from src.submission.check_validity import get_model_size - -EVAL_REQUESTS_PATH = "eval-queue" -QUEUE_REPO = "open-llm-leaderboard/requests" - -precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ") -model_types = [e.name for e in ModelType] -weight_types = [e.name for e in WeightType] - - -def main(): - api = HfApi() - current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset") - - model_name = click.prompt("Enter model name") - revision = click.prompt("Enter revision", default="main") - precision = click.prompt("Enter precision", default="float16", type=click.Choice(precisions)) - model_type = click.prompt("Enter model type", type=click.Choice(model_types)) - weight_type = click.prompt("Enter weight type", default="Original", type=click.Choice(weight_types)) - base_model = click.prompt("Enter base model", default="") - status = click.prompt("Enter status", default="FINISHED") - - try: - model_info = api.model_info(repo_id=model_name, revision=revision) - except Exception as e: - print(f"{Fore.RED}Could not find model info for {model_name} on the Hub\n{e}{Fore.RESET}") - return 1 - - model_size = get_model_size(model_info=model_info, precision=precision) - - try: - license = model_info.cardData["license"] - except Exception: - license = "?" - - eval_entry = { - "model": model_name, - "base_model": base_model, - "revision": model_info.sha, # force to use the exact model commit - "private": False, - "precision": precision, - "weight_type": weight_type, - "status": status, - "submitted_time": current_time, - "model_type": model_type, - "likes": model_info.likes, - "params": model_size, - "license": license, - } - - user_name = "" - model_path = model_name - if "/" in model_name: - user_name = model_name.split("/")[0] - model_path = model_name.split("/")[1] - - pprint.pprint(eval_entry) - - if click.confirm("Do you want to continue? This request file will be pushed to the hub"): - click.echo("continuing...") - - out_dir = f"{EVAL_REQUESTS_PATH}/{user_name}" - os.makedirs(out_dir, exist_ok=True) - out_path = f"{out_dir}/{model_path}_eval_request_{False}_{precision}_{weight_type}.json" - - with open(out_path, "w") as f: - f.write(json.dumps(eval_entry)) - - api.upload_file( - path_or_fileobj=out_path, - path_in_repo=out_path.split(f"{EVAL_REQUESTS_PATH}/")[1], - repo_id=QUEUE_REPO, - repo_type="dataset", - commit_message=f"Add {model_name} to eval queue", - ) - else: - click.echo("aborting...") - - -if __name__ == "__main__": - main() diff --git a/src/tools/model_backlinks.py b/src/tools/model_backlinks.py deleted file mode 100644 index c1afa53b0549a15d7f384621e3c0655aad67f67e..0000000000000000000000000000000000000000 --- a/src/tools/model_backlinks.py +++ /dev/null @@ -1,1309 +0,0 @@ -models = [ - "uni-tianyan/Uni-TianYan", - "fangloveskari/ORCA_LLaMA_70B_QLoRA", - "garage-bAInd/Platypus2-70B-instruct", - "upstage/Llama-2-70b-instruct-v2", - "fangloveskari/Platypus_QLoRA_LLaMA_70b", - "yeontaek/llama-2-70B-ensemble-v5", - "TheBloke/Genz-70b-GPTQ", - "TheBloke/Platypus2-70B-Instruct-GPTQ", - "psmathur/model_007", - "yeontaek/llama-2-70B-ensemble-v4", - "psmathur/orca_mini_v3_70b", - "ehartford/Samantha-1.11-70b", - "MayaPH/GodziLLa2-70B", - "psmathur/model_007_v2", - "chargoddard/MelangeA-70b", - "ehartford/Samantha-1.1-70b", - "psmathur/model_009", - "upstage/Llama-2-70b-instruct", - "yeontaek/llama-2-70B-ensemble-v7", - "yeontaek/llama-2-70B-ensemble-v6", - "chargoddard/MelangeB-70b", - "yeontaek/llama-2-70B-ensemble-v3", - "chargoddard/MelangeC-70b", - "garage-bAInd/Camel-Platypus2-70B", - "yeontaek/llama-2-70B-ensemble-v2", - "garage-bAInd/Camel-Platypus2-70B", - "migtissera/Synthia-70B-v1.2", - "v2ray/LLaMA-2-Wizard-70B-QLoRA", - "quantumaikr/llama-2-70b-fb16-orca-chat-10k", - "v2ray/LLaMA-2-Wizard-70B-QLoRA", - "stabilityai/StableBeluga2", - "quantumaikr/llama-2-70b-fb16-guanaco-1k", - "garage-bAInd/Camel-Platypus2-70B", - "migtissera/Synthia-70B-v1.1", - "migtissera/Synthia-70B", - "psmathur/model_101", - "augtoma/qCammel70", - "augtoma/qCammel-70", - "augtoma/qCammel-70v1", - "augtoma/qCammel-70x", - "augtoma/qCammel-70-x", - "jondurbin/airoboros-l2-70b-gpt4-1.4.1", - "dfurman/llama-2-70b-dolphin-peft", - "jondurbin/airoboros-l2-70b-2.1", - "TheBloke/llama-2-70b-Guanaco-QLoRA-fp16", - "quantumaikr/QuantumLM-llama2-70B-Korean-LoRA", - "quantumaikr/quantumairk-llama-2-70B-instruct", - "psmathur/model_420", - "psmathur/model_51", - "garage-bAInd/Camel-Platypus2-70B", - "TheBloke/Airoboros-L2-70B-2.1-GPTQ", - "OpenAssistant/llama2-70b-oasst-sft-v10", - "garage-bAInd/Platypus2-70B", - "liuxiang886/llama2-70B-qlora-gpt4", - "upstage/llama-65b-instruct", - "quantumaikr/llama-2-70b-fb16-korean", - "NousResearch/Nous-Hermes-Llama2-70b", - "v2ray/LLaMA-2-Jannie-70B-QLoRA", - "jondurbin/airoboros-l2-70b-gpt4-m2.0", - "jondurbin/airoboros-l2-70b-gpt4-m2.0", - "OpenAssistant/llama2-70b-oasst-sft-v10", - "yeontaek/llama-2-70B-ensemble-v8", - "jondurbin/airoboros-l2-70b-gpt4-2.0", - "jarradh/llama2_70b_chat_uncensored", - "WizardLM/WizardMath-70B-V1.0", - "jordiclive/Llama-2-70b-oasst-1-200", - "WizardLM/WizardMath-70B-V1.0", - "jondurbin/airoboros-l2-70b-gpt4-2.0", - "OpenLemur/lemur-70b-chat-v1", - "tiiuae/falcon-180B", - "tiiuae/falcon-180B", - "stabilityai/StableBeluga1-Delta", - "psmathur/model_42_70b", - "psmathur/test_42_70b", - "TheBloke/fiction.live-Kimiko-V2-70B-fp16", - "tiiuae/falcon-180B", - "WizardLM/WizardMath-70B-V1.0", - "tiiuae/falcon-180B-chat", - "jondurbin/airoboros-l2-70b-gpt4-2.0", - "ehartford/samantha-1.1-llama-33b", - "ajibawa-2023/scarlett-33b", - "ddobokki/Llama-2-70b-orca-200k", - "TheBloke/gpt4-alpaca-lora_mlp-65B-HF", - "tiiuae/falcon-180B-chat", - "tiiuae/falcon-180B-chat", - "tiiuae/falcon-180B", - "TheBloke/Lemur-70B-Chat-v1-GPTQ", - "NousResearch/Nous-Puffin-70B", - "WizardLM/WizardLM-70B-V1.0", - "WizardLM/WizardMath-70B-V1.0", - "meta-llama/Llama-2-70b-hf", - "TheBloke/Llama-2-70B-fp16", - "Weyaxi/llama-2-alpacagpt4-1000step", - "WizardLM/WizardLM-70B-V1.0", - "simsim314/WizardLM-70B-V1.0-HF", - "simsim314/WizardLM-70B-V1.0-HF", - "WizardLM/WizardLM-70B-V1.0", - "openbmb/UltraLM-65b", - "psmathur/model_420_preview", - "WizardLM/WizardLM-70B-V1.0", - "simsim314/WizardLM-70B-V1.0-HF", - "OpenBuddy/openbuddy-llama2-70b-v10.1-bf16", - "upstage/llama-30b-instruct-2048", - "jondurbin/airoboros-65b-gpt4-1.2", - "TheBloke/guanaco-65B-HF", - "jondurbin/airoboros-65b-gpt4-1.3", - "meta-llama/Llama-2-70b-chat-hf", - "ValiantLabs/ShiningValiant", - "Faradaylab/Aria-70B", - "lilloukas/GPlatty-30B", - "TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16", - "jondurbin/airoboros-65b-gpt4-1.4-peft", - "jondurbin/airoboros-65b-gpt4-1.4", - "jondurbin/airoboros-65b-gpt4-2.0", - "TheBloke/WizardLM-70B-V1.0-GPTQ", - "TheBloke/WizardLM-70B-V1.0-GPTQ", - "ariellee/SuperPlatty-30B", - "jondurbin/airoboros-65b-gpt4-1.4", - "jondurbin/airoboros-65b-gpt4-2.0", - "yeontaek/llama-2-70b-IA3-guanaco", - "CalderaAI/30B-Lazarus", - "Aspik101/trurl-2-13b-pl-instruct_unload", - "ehartford/WizardLM-33B-V1.0-Uncensored", - "ehartford/WizardLM-33B-V1.0-Uncensored", - "OpenBuddy/openbuddy-llama-65b-v8-bf16", - "Aspik101/llama-30b-instruct-2048-PL-lora", - "h2oai/h2ogpt-research-oasst1-llama-65b", - "Aspik101/llama-30b-instruct-2048-PL-lora", - "CalderaAI/30B-Epsilon", - "Aspik101/llama-30b-2048-instruct-PL-lora_unload", - "jondurbin/airoboros-65b-gpt4-m2.0", - "jondurbin/airoboros-65b-gpt4-m2.0", - "Aeala/Alpaca-elina-65b", - "TheBloke/robin-65b-v2-fp16", - "TheBloke/gpt4-alpaca-lora-30b-HF", - "TheBloke/Llama-2-70B-chat-GPTQ", - "upstage/llama-30b-instruct", - "OpenLemur/lemur-70b-v1", - "lmsys/vicuna-33b-v1.3", - "ausboss/llama-30b-supercot", - "ai-business/Luban-13B", - "Henk717/airochronos-33B", - "lmsys/vicuna-33b-v1.3", - "Henk717/airochronos-33B", - "bavest/fin-llama-33b-merged", - "jondurbin/airoboros-33b-gpt4-1.4", - "YeungNLP/firefly-llama-30b", - "Aspik101/30B-Lazarus-instruct-PL-lora_unload", - "uukuguy/speechless-llama2-luban-orca-platypus-13b", - "xxyyy123/test_merge_p_ov1_w0.66_w0.5_n1", - "jondurbin/airoboros-33b-gpt4-1.2", - "TheBloke/alpaca-lora-65B-HF", - "bofenghuang/vigogne-33b-instruct", - "yeontaek/llama-2-13B-ensemble-v5", - "garage-bAInd/Platypus-30B", - "Open-Orca/OpenOrca-Platypus2-13B", - "kajdun/viwaai-30b_v4", - "lilloukas/Platypus-30B", - "Open-Orca/OpenOrca-Platypus2-13B", - "Henk717/chronoboros-33B", - "jondurbin/airoboros-33b-2.1", - "HiTZ/alpaca-lora-65b-en-pt-es-ca", - "quantumaikr/QuantumLM-70B-hf", - "uukuguy/speechless-llama2-13b", - "uukuguy/speechless-llama2-hermes-orca-platypus-13b", - "openaccess-ai-collective/manticore-30b-chat-pyg-alpha", - "LLMs/WizardLM-30B-V1.0", - "TheBloke/WizardLM-30B-fp16", - "openaccess-ai-collective/hippogriff-30b-chat", - "concedo/Vicuzard-30B-Uncensored", - "TFLai/OpenOrca-Platypus2-13B-QLoRA-0.80-epoch", - "huggingface/llama-65b", - "huggyllama/llama-65b", - "gaodrew/gaodrew-llama-30b-instruct-2048-Open-Platypus-100steps", - "uukuguy/speechless-llama2-hermes-orca-platypus-wizardlm-13b", - "Sao10K/Mythical-Destroyer-V2-L2-13B", - "camel-ai/CAMEL-33B-Combined-Data", - "dsvv-cair/alpaca-cleaned-llama-30b-bf16", - "MetaIX/GPT4-X-Alpasta-30b", - "garage-bAInd/Stable-Platypus2-13B", - "TFLai/Luban-Platypus2-13B-QLora-0.80-epoch", - "TheBloke/OpenOrca-Platypus2-13B-GPTQ", - "IkariDev/Athena-tmp", - "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16", - "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16", - "Open-Orca/OpenOrcaxOpenChat-Preview2-13B", - "psmathur/model_007_13b_v2", - "Aspik101/Vicuzard-30B-Uncensored-instruct-PL-lora_unload", - "jondurbin/airoboros-33b-gpt4-m2.0", - "Sao10K/Mythical-Destroyer-L2-13B", - "TheBloke/Wizard-Vicuna-30B-Uncensored-fp16", - "ehartford/Wizard-Vicuna-30B-Uncensored", - "TFLai/Nova-13B", - "TheBloke/robin-33B-v2-fp16", - "totally-not-an-llm/PuddleJumper-13b", - "Aeala/VicUnlocked-alpaca-30b", - "Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf", - "jondurbin/airoboros-33b-gpt4", - "jondurbin/airoboros-33b-gpt4-m2.0", - "tiiuae/falcon-40b-instruct", - "psmathur/orca_mini_v3_13b", - "Aeala/GPT4-x-AlpacaDente-30b", - "MayaPH/GodziLLa-30B", - "jondurbin/airoboros-33b-gpt4-m2.0", - "TFLai/SpeechlessV1-Nova-13B", - "yeontaek/llama-2-13B-ensemble-v4", - "ajibawa-2023/carl-33b", - "jondurbin/airoboros-33b-gpt4-2.0", - "TFLai/Stable-Platypus2-13B-QLoRA-0.80-epoch", - "jondurbin/airoboros-33b-gpt4-1.3", - "TehVenom/oasst-sft-6-llama-33b-xor-MERGED-16bit", - "TFLai/OrcaMini-Platypus2-13B-QLoRA-0.80-epoch", - "jondurbin/airoboros-33b-gpt4-2.0", - "chargoddard/Chronorctypus-Limarobormes-13b", - "jondurbin/airoboros-33b-gpt4-1.3", - "Open-Orca/OpenOrca-Platypus2-13B", - "FelixChao/vicuna-33b-coder", - "FelixChao/vicuna-33b-coder", - "Gryphe/MythoMix-L2-13b", - "Aeala/Enterredaas-33b", - "yeontaek/llama-2-13B-ensemble-v1", - "TFLai/OpenOrcaPlatypus2-Platypus2-13B-QLora-0.80-epoch", - "TFLai/Ensemble5-Platypus2-13B-QLora-0.80-epoch", - "yeontaek/llama-2-13B-ensemble-v3", - "TFLai/MythoMix-Platypus2-13B-QLoRA-0.80-epoch", - "yihan6324/llama2-13b-instructmining-40k-sharegpt", - "timdettmers/guanaco-33b-merged", - "TFLai/EnsembleV5-Nova-13B", - "circulus/Llama-2-13b-orca-v1", - "Undi95/ReMM-SLERP-L2-13B", - "Gryphe/MythoMax-L2-13b", - "stabilityai/StableBeluga-13B", - "circulus/Llama-2-13b-orca-v1", - "ehartford/WizardLM-30B-Uncensored", - "The-Face-Of-Goonery/huginnv1.2", - "TheBloke/OpenOrcaxOpenChat-Preview2-13B-GPTQ", - "Sao10K/Stheno-L2-13B", - "bofenghuang/vigogne-2-13b-instruct", - "The-Face-Of-Goonery/Huginn-13b-FP16", - "grimpep/L2-MythoMax22b-instruct-Falseblock", - "TFLai/Nous-Hermes-Platypus2-13B-QLoRA-0.80-epoch", - "yeontaek/Platypus2xOpenOrca-13B-IA3-v4", - "yeontaek/Platypus2xOpenOrca-13B-IA3", - "yeontaek/Platypus2xOpenOrca-13B-IA3-ensemble", - "Open-Orca/LlongOrca-13B-16k", - "Sao10K/Stheno-Inverted-L2-13B", - "garage-bAInd/Camel-Platypus2-13B", - "digitous/Alpacino30b", - "NousResearch/Nous-Hermes-Llama2-13b", - "yeontaek/Platypus2xOpenOrca-13B-IA3-v3", - "TFLai/MythicalDestroyerV2-Platypus2-13B-QLora-0.80-epoch", - "TheBloke/VicUnlocked-30B-LoRA-HF", - "Undi95/Nous-Hermes-13B-Code", - "The-Face-Of-Goonery/Chronos-Beluga-v2-13bfp16", - "NousResearch/Nous-Hermes-Llama2-13b", - "Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b", - "TheBloke/Wizard-Vicuna-30B-Uncensored-GPTQ", - "Open-Orca/OpenOrcaxOpenChat-Preview2-13B", - "Austism/chronos-hermes-13b-v2", - "yeontaek/Platypus2xOpenOrca-13B-IA3-v2.1", - "yeontaek/Platypus2xOpenOrca-13B-IA3-v2", - "Gryphe/MythoLogic-L2-13b", - "augtoma/qCammel-13", - "YeungNLP/firefly-llama2-13b-v1.2", - "Aspik101/StableBeluga-13B-instruct-PL-lora_unload", - "andreaskoepf/llama2-13b-megacode2_min100", - "rombodawg/LosslessMegaCoder-llama2-13b-mini", - "yulan-team/YuLan-Chat-2-13b-fp16", - "elinas/chronos-33b", - "YeungNLP/firefly-llama2-13b", - "Sao10K/Medusa-13b", - "OptimalScale/robin-65b-v2-delta", - "minlik/chinese-alpaca-33b-merged", - "OpenAssistant/llama2-13b-megacode2-oasst", - "TheBloke/OpenAssistant-SFT-7-Llama-30B-HF", - "Undi95/UndiMix-v1-13b", - "ehartford/Samantha-1.11-13b", - "beaugogh/Llama2-13b-sharegpt4", - "Aeala/GPT4-x-AlpacaDente2-30b", - "luffycodes/nash-vicuna-13b-v1dot5-ep2-w-rag-w-simple", - "WizardLM/WizardLM-13B-V1.1", - "uukuguy/speechless-orca-platypus-coig-lite-2k-0.6e-13b", - "huggyllama/llama-30b", - "Undi95/ReMM-L2-13B-PIPPA", - "Undi95/ReMM-L2-13B", - "gaodrew/gaodrew-gorgonzola-13b", - "lmsys/vicuna-13b-v1.5", - "yeontaek/Platypus2xOpenOrca-13B-LoRa", - "Yhyu13/llama-30B-hf-openassitant", - "huggingface/llama-30b", - "lmsys/vicuna-13b-v1.5", - "TFLai/Athena-Platypus2-13B-QLora-0.80-epoch", - "TheBloke/dromedary-65b-lora-HF", - "yeontaek/llama-2-13b-Beluga-QLoRA", - "The-Face-Of-Goonery/Huginn-13b-V4", - "The-Face-Of-Goonery/Huginn-13b-v4.5", - "The-Face-Of-Goonery/Huginn-v3-13b", - "tiiuae/falcon-40b", - "WhoTookMyAmogusNickname/NewHope_HF_not_official", - "gaodrew/OpenOrca-Platypus2-13B-thera-1250", - "SLAM-group/NewHope", - "garage-bAInd/Platypus2-13B", - "migtissera/Synthia-13B", - "elinas/chronos-13b-v2", - "mosaicml/mpt-30b-chat", - "CHIH-HUNG/llama-2-13b-OpenOrca_5w", - "uukuguy/speechless-hermes-coig-lite-13b", - "TheBloke/tulu-30B-fp16", - "uukuguy/speechless-hermes-coig-lite-13b", - "xDAN-AI/xDAN_13b_l2_lora", - "lmsys/vicuna-13b-v1.5-16k", - "openchat/openchat_v3.1", - "CHIH-HUNG/llama-2-13b-dolphin_5w", - "Aspik101/vicuna-13b-v1.5-PL-lora_unload", - "Undi95/MLewd-L2-13B", - "ehartford/minotaur-llama2-13b-qlora", - "kajdun/iubaris-13b-v3", - "TFLai/Limarp-Platypus2-13B-QLoRA-0.80-epoch", - "openchat/openchat_v3.1", - "uukuguy/speechless-orca-platypus-coig-lite-4k-0.6e-13b", - "ziqingyang/chinese-alpaca-2-13b", - "TFLai/Airboros2.1-Platypus2-13B-QLora-0.80-epoch", - "yeontaek/llama-2-13b-Guanaco-QLoRA", - "lmsys/vicuna-13b-v1.5-16k", - "ehartford/based-30b", - "kingbri/airolima-chronos-grad-l2-13B", - "openchat/openchat_v3.2", - "uukuguy/speechless-orca-platypus-coig-lite-4k-0.5e-13b", - "yeontaek/Platypus2-13B-LoRa", - "kingbri/chronolima-airo-grad-l2-13B", - "openchat/openchat_v3.2", - "TFLai/PuddleJumper-Platypus2-13B-QLoRA-0.80-epoch", - "shareAI/llama2-13b-Chinese-chat", - "ehartford/WizardLM-1.0-Uncensored-Llama2-13b", - "Aspik101/Redmond-Puffin-13B-instruct-PL-lora_unload", - "yeontaek/llama-2-13B-ensemble-v6", - "WizardLM/WizardLM-13B-V1.2", - "TheBloke/WizardLM-13B-V1.1-GPTQ", - "bhenrym14/airophin-13b-pntk-16k-fp16", - "ehartford/WizardLM-1.0-Uncensored-Llama2-13b", - "Mikael110/llama-2-13b-guanaco-fp16", - "yeontaek/airoboros-2.1-llama-2-13B-QLoRa", - "CalderaAI/13B-Legerdemain-L2", - "grimpep/llama2-22b-wizard_vicuna", - "grimpep/llama2-22B-GPLATTY", - "bhenrym14/airophin-13b-pntk-16k-fp16", - "yeontaek/llama-2-13b-QLoRA", - "OpenAssistant/llama2-13b-orca-8k-3319", - "TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16", - "duliadotio/dulia-13b-8k-alpha", - "Undi95/LewdEngine", - "OpenBuddy/openbuddy-llama2-13b-v8.1-fp16", - "CHIH-HUNG/llama-2-13b-open_orca_20w", - "bhenrym14/airoboros-33b-gpt4-1.4.1-lxctx-PI-16384-fp16", - "FlagAlpha/Llama2-Chinese-13b-Chat", - "LLMs/WizardLM-13B-V1.0", - "chansung/gpt4-alpaca-lora-13b-decapoda-1024", - "TheBloke/wizardLM-13B-1.0-fp16", - "digitous/13B-Chimera", - "yeontaek/Platypus2xOpenOrcaxGuanaco-13B-LoRa", - "jondurbin/airoboros-l2-13b-2.1", - "Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b", - "TheBloke/UltraLM-13B-fp16", - "openaccess-ai-collective/minotaur-13b-fixed", - "NousResearch/Redmond-Puffin-13B", - "KoboldAI/LLaMA2-13B-Holomax", - "Lajonbot/WizardLM-13B-V1.2-PL-lora_unload", - "yeontaek/Platypus2-13B-LoRa-v2", - "TheBloke/airoboros-13B-HF", - "jondurbin/airoboros-13b", - "jjaaaww/posi_13b", - "CoolWP/llama-2-13b-guanaco-fp16", - "yeontaek/Platypus2-13B-QLoRa", - "h2oai/h2ogpt-research-oig-oasst1-512-30b", - "dfurman/llama-2-13b-guanaco-peft", - "NousResearch/Redmond-Puffin-13B", - "pe-nlp/llama-2-13b-platypus-vicuna-wizard", - "CHIH-HUNG/llama-2-13b-dolphin_20w", - "NousResearch/Nous-Hermes-13b", - "NobodyExistsOnTheInternet/GiftedConvo13bLoraNoEconsE4", - "ehartford/Wizard-Vicuna-13B-Uncensored", - "TheBloke/Wizard-Vicuna-13B-Uncensored-HF", - "openchat/openchat_v3.2_super", - "bhenrym14/airophin-v2-13b-PI-8k-fp16", - "openaccess-ai-collective/manticore-13b", - "The-Face-Of-Goonery/Huginn-22b-Prototype", - "jphme/Llama-2-13b-chat-german", - "grimpep/llama2-28B-Airo03", - "TheBloke/Kimiko-v2-13B-fp16", - "FPHam/Free_Sydney_13b_HF", - "lmsys/vicuna-13b-v1.3", - "FelixChao/llama2-13b-math1.1", - "CalderaAI/13B-BlueMethod", - "meta-llama/Llama-2-13b-chat-hf", - "deepse/CodeUp-Llama-2-13b-chat-hf", - "WizardLM/WizardMath-13B-V1.0", - "WizardLM/WizardMath-13B-V1.0", - "HyperbeeAI/Tulpar-7b-v0", - "xxyyy123/test_qkvo_adptor", - "xxyyy123/mc_data_30k_from_platpus_orca_7b_10k_v1_lora_qkvo_rank14_v2", - "openchat/openchat_v2_w", - "FelixChao/llama2-13b-math1.1", - "psmathur/orca_mini_v3_7b", - "TehVenom/Metharme-13b-Merged", - "xxyyy123/10k_v1_lora_qkvo_rank14_v3", - "OpenAssistant/llama2-13b-orca-v2-8k-3166", - "openaccess-ai-collective/wizard-mega-13b", - "jondurbin/airoboros-13b-gpt4-1.4", - "jondurbin/airoboros-13b-gpt4-1.4-fp16", - "Monero/Manticore-13b-Chat-Pyg-Guanaco", - "FelixChao/llama2-13b-math1.2", - "chargoddard/platypus-2-22b-relora", - "FelixChao/llama2-13b-math1.2", - "Gryphe/MythoBoros-13b", - "CalderaAI/13B-Ouroboros", - "OpenAssistant/llama2-13b-orca-v2-8k-3166", - "heegyu/LIMA2-13b-hf", - "digitous/13B-HyperMantis", - "Gryphe/MythoLogic-13b", - "TheBloke/Airoboros-L2-13B-2.1-GPTQ", - "chargoddard/platypus2-22b-relora", - "openchat/openchat_v2", - "yeontaek/Platypus2-13B-IA3", - "stabilityai/StableBeluga-7B", - "circulus/Llama-2-7b-orca-v1", - "budecosystem/genz-13b-v2", - "TheBloke/gpt4-x-vicuna-13B-HF", - "NobodyExistsOnTheInternet/GiftedConvo13bLoraNoEcons", - "zarakiquemparte/zarafusionex-1.1-l2-7b", - "Lajonbot/tableBeluga-7B-instruct-pl-lora_unload", - "jondurbin/airoboros-13b-gpt4", - "gaodrew/gaodrew-gorgonzola-13b", - "jondurbin/airoboros-13b-gpt4-1.1", - "TheBloke/gpt4-alpaca-lora-13B-HF", - "zarakiquemparte/zarablendex-vq-l2-7b", - "openaccess-ai-collective/manticore-13b-chat-pyg", - "Lajonbot/Llama-2-13b-hf-instruct-pl-lora_unload", - "NobodyExistsOnTheInternet/PuffedLIMA13bQLORA", - "xxyyy123/10k_v1_lora_qkvo_rank28_v2", - "jondurbin/airoboros-l2-13b-gpt4-1.4.1", - "dhmeltzer/Llama-2-13b-hf-eli5-wiki-1024_r_64_alpha_16", - "NobodyExistsOnTheInternet/PuffedConvo13bLoraE4", - "yihan6324/llama2-7b-instructmining-40k-sharegpt", - "CHIH-HUNG/llama-2-13b-Open_Platypus_and_ccp_2.6w", - "Aeala/GPT4-x-Alpasta-13b", - "psmathur/orca_mini_v2_13b", - "YeungNLP/firefly-llama-13b", - "psmathur/orca_mini_v2_13b", - "zarakiquemparte/zarafusionix-l2-7b", - "yihan6324/llama2-7b-instructmining-60k-sharegpt", - "yihan6324/llama-2-7b-instructmining-60k-sharegpt", - "layoric/llama-2-13b-code-alpaca", - "bofenghuang/vigogne-13b-instruct", - "Lajonbot/vicuna-13b-v1.3-PL-lora_unload", - "lvkaokao/llama2-7b-hf-chat-lora-v3", - "ehartford/dolphin-llama-13b", - "YeungNLP/firefly-llama-13b-v1.2", - "TheBloke/Kimiko-13B-fp16", - "kevinpro/Vicuna-13B-CoT", - "eachadea/vicuna-13b-1.1", - "pillowtalks-ai/delta13b", - "TheBloke/vicuna-13B-1.1-HF", - "TheBloke/Vicuna-13B-CoT-fp16", - "lmsys/vicuna-13b-delta-v1.1", - "lmsys/vicuna-13b-v1.1", - "xxyyy123/20k_v1_lora_qkvo_rank14_v2", - "TheBloke/guanaco-13B-HF", - "TheBloke/vicuna-13b-v1.3.0-GPTQ", - "edor/Stable-Platypus2-mini-7B", - "totally-not-an-llm/EverythingLM-13b-V2-16k", - "zarakiquemparte/zaraxe-l2-7b", - "beaugogh/Llama2-7b-openorca-mc-v2", - "TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16", - "quantumaikr/QuantumLM", - "jondurbin/airoboros-13b-gpt4-1.2", - "TheBloke/robin-13B-v2-fp16", - "TFLai/llama-2-13b-4bit-alpaca-gpt4", - "yihan6324/llama2-7b-instructmining-orca-40k", - "dvruette/oasst-llama-13b-2-epochs", - "Open-Orca/LlongOrca-7B-16k", - "Aspik101/Nous-Hermes-13b-pl-lora_unload", - "ehartford/Samantha-1.11-CodeLlama-34b", - "nkpz/llama2-22b-chat-wizard-uncensored", - "bofenghuang/vigogne-13b-chat", - "beaugogh/Llama2-7b-openorca-mc-v1", - "OptimalScale/robin-13b-v2-delta", - "pe-nlp/llama-2-13b-vicuna-wizard", - "chargoddard/llama2-22b", - "gywy/llama2-13b-chinese-v1", - "frank098/Wizard-Vicuna-13B-juniper", - "IGeniusDev/llama13B-quant8-testv1-openorca-customdataset", - "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w-gate_up_down_proj", - "eachadea/vicuna-13b", - "yihan6324/llama2-7b-instructmining-orca-90k", - "chargoddard/llama2-22b-blocktriangular", - "luffycodes/mcq-vicuna-13b-v1.5", - "Yhyu13/chimera-inst-chat-13b-hf", - "luffycodes/mcq-vicuna-13b-v1.5", - "chargoddard/ypotryll-22b-epoch2-qlora", - "totally-not-an-llm/EverythingLM-13b-16k", - "luffycodes/mcq-hal-vicuna-13b-v1.5", - "openaccess-ai-collective/minotaur-13b", - "IGeniusDev/llama13B-quant8-testv1-openorca-customdataset", - "chargoddard/llama2-22b-blocktriangular", - "TFLai/Platypus2-13B-QLoRA-0.80-epoch", - "meta-llama/Llama-2-13b-hf", - "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w-gate_up_down_proj", - "luffycodes/mcq-hal-vicuna-13b-v1.5", - "TheBloke/Llama-2-13B-fp16", - "TaylorAI/Flash-Llama-13B", - "shareAI/bimoGPT-llama2-13b", - "wahaha1987/llama_13b_sharegpt94k_fastchat", - "openchat/openchat_8192", - "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w-q_k_v_o_proj", - "dvruette/llama-13b-pretrained-sft-do2", - "CHIH-HUNG/llama-2-13b-alpaca-test", - "OpenBuddy/openbuddy-llama2-13b-v11.1-bf16", - "CHIH-HUNG/llama-2-13b-FINETUNE2_TEST_2.2w", - "project-baize/baize-v2-13b", - "jondurbin/airoboros-l2-13b-gpt4-m2.0", - "yeontaek/Platypus2xOpenOrca-13B-LoRa-v2", - "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w", - "xzuyn/Alpacino-SuperCOT-13B", - "jondurbin/airoboros-l2-13b-gpt4-2.0", - "aiplanet/effi-13b", - "clibrain/Llama-2-13b-ft-instruct-es", - "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w", - "bofenghuang/vigogne-2-7b-instruct", - "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w-q_k_v_o_proj", - "bofenghuang/vigogne-2-7b-chat", - "aiplanet/effi-13b", - "haonan-li/bactrian-x-llama-13b-merged", - "beaugogh/Llama2-7b-sharegpt4", - "HWERI/Llama2-7b-sharegpt4", - "jondurbin/airoboros-13b-gpt4-1.3", - "jondurbin/airoboros-c34b-2.1", - "junelee/wizard-vicuna-13b", - "TheBloke/wizard-vicuna-13B-HF", - "Open-Orca/OpenOrca-Preview1-13B", - "TheBloke/h2ogpt-oasst1-512-30B-HF", - "TheBloke/Llama-2-13B-GPTQ", - "camel-ai/CAMEL-13B-Combined-Data", - "lmsys/vicuna-7b-v1.5", - "lmsys/vicuna-7b-v1.5-16k", - "lmsys/vicuna-7b-v1.5", - "ausboss/llama-13b-supercot", - "TheBloke/tulu-13B-fp16", - "NousResearch/Nous-Hermes-llama-2-7b", - "jlevin/guanaco-13b-llama-2", - "lmsys/vicuna-7b-v1.5-16k", - "dvruette/llama-13b-pretrained", - "nkpz/llama2-22b-daydreamer-v3", - "dvruette/llama-13b-pretrained-dropout", - "jondurbin/airoboros-l2-13b-2.1", - "LLMs/Stable-Vicuna-13B", - "64bits/LexPodLM-13B", - "lizhuang144/llama_mirror_13b_v1.0", - "TheBloke/stable-vicuna-13B-HF", - "zarakiquemparte/zaraxls-l2-7b", - "TheBloke/Llama-2-13B-GPTQ", - "Kiddyz/testlm-3", - "migtissera/Synthia-7B", - "zarakiquemparte/zarablend-l2-7b", - "mosaicml/mpt-30b-instruct", - "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged", - "vonjack/Qwen-LLaMAfied-HFTok-7B-Chat", - "l3utterfly/llama2-7b-layla", - "Lajonbot/vicuna-7b-v1.5-PL-lora_unload", - "heegyu/LIMA-13b-hf", - "frank098/WizardLM_13B_juniper", - "ashercn97/manatee-7b", - "chavinlo/gpt4-x-alpaca", - "PocketDoc/Dans-PersonalityEngine-13b", - "ehartford/WizardLM-1.0-Uncensored-CodeLlama-34b", - "digitous/Alpacino13b", - "edor/Hermes-Platypus2-mini-7B", - "lvkaokao/llama2-7b-hf-chat-lora-v2", - "Kiddyz/testlm-1-1", - "Kiddyz/testlm", - "Kiddyz/testlm-1", - "Kiddyz/testlm2", - "radm/Philosophy-Platypus2-13b", - "aiplanet/effi-13b", - "Harshvir/Llama-2-7B-physics", - "YeungNLP/firefly-ziya-13b", - "LinkSoul/Chinese-Llama-2-7b", - "PeanutJar/LLaMa-2-PeanutButter_v10-7B", - "OpenBuddy/openbuddy-llama2-13b-v11-bf16", - "StudentLLM/Alpagasus-2-13B-QLoRA-pipeline", - "meta-llama/Llama-2-13b-hf", - "WizardLM/WizardCoder-Python-34B-V1.0", - "dvruette/llama-13b-pretrained-sft-epoch-1", - "camel-ai/CAMEL-13B-Role-Playing-Data", - "ziqingyang/chinese-llama-2-13b", - "rombodawg/LosslessMegaCoder-llama2-7b-mini", - "TheBloke/koala-13B-HF", - "lmsys/vicuna-7b-delta-v1.1", - "eachadea/vicuna-7b-1.1", - "Ejafa/vicuna_7B_vanilla_1.1", - "lvkaokao/llama2-7b-hf-chat-lora", - "OpenBuddy/openbuddy-atom-13b-v9-bf16", - "Norquinal/llama-2-7b-claude-chat-rp", - "Danielbrdz/Barcenas-7b", - "heegyu/WizardVicuna2-13b-hf", - "meta-llama/Llama-2-7b-chat-hf", - "PeanutJar/LLaMa-2-PeanutButter_v14-7B", - "PeanutJar/LLaMa-2-PeanutButter_v4-7B", - "davzoku/cria-llama2-7b-v1.3", - "OpenBuddy/openbuddy-atom-13b-v9-bf16", - "lvkaokao/llama2-7b-hf-instruction-lora", - "Tap-M/Luna-AI-Llama2-Uncensored", - "ehartford/Samantha-1.11-7b", - "WizardLM/WizardCoder-Python-34B-V1.0", - "TheBloke/Manticore-13B-Chat-Pyg-Guanaco-SuperHOT-8K-GPTQ", - "Mikael110/llama-2-7b-guanaco-fp16", - "garage-bAInd/Platypus2-7B", - "PeanutJar/LLaMa-2-PeanutButter_v18_B-7B", - "mosaicml/mpt-30b", - "garage-bAInd/Platypus2-7B", - "huggingface/llama-13b", - "dvruette/oasst-llama-13b-1000-steps", - "jordiclive/gpt4all-alpaca-oa-codealpaca-lora-13b", - "huggyllama/llama-13b", - "Voicelab/trurl-2-7b", - "TFLai/llama-13b-4bit-alpaca", - "gywy/llama2-13b-chinese-v2", - "lmsys/longchat-13b-16k", - "Aspik101/trurl-2-7b-pl-instruct_unload", - "WizardLM/WizardMath-7B-V1.0", - "Norquinal/llama-2-7b-claude-chat", - "TheTravellingEngineer/llama2-7b-chat-hf-dpo", - "open-llm-leaderboard/starchat-beta", - "joehuangx/spatial-vicuna-7b-v1.5-LoRA", - "conceptofmind/LLongMA-2-13b-16k", - "tianyil1/denas-llama2", - "lmsys/vicuna-7b-v1.3", - "conceptofmind/LLongMA-2-13b-16k", - "openchat/opencoderplus", - "ajibawa-2023/scarlett-7b", - "dhmeltzer/llama-7b-SFT_eli5_wiki65k_1024_r_64_alpha_16_merged", - "psyche/kollama2-7b-v2", - "heegyu/LIMA2-7b-hf", - "dhmeltzer/llama-7b-SFT-qlora-eli5-wiki_DPO_ds_RM_top_2_1024_r_64_alpha_16", - "abhishek/llama2guanacotest", - "jondurbin/airoboros-l2-7b-2.1", - "llama-anon/instruct-13b", - "FelixChao/vicuna-7B-physics", - "Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload", - "shibing624/chinese-alpaca-plus-13b-hf", - "davzoku/cria-llama2-7b-v1.3_peft", - "quantumaikr/llama-2-7b-hf-guanaco-1k", - "togethercomputer/Llama-2-7B-32K-Instruct", - "sia-ai/llama-2-7b-1-percent-open-orca-1000-steps-v0", - "TheTravellingEngineer/llama2-7b-hf-guanaco", - "Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload", - "jondurbin/airoboros-l2-7b-gpt4-1.4.1", - "wahaha1987/llama_7b_sharegpt94k_fastchat", - "FelixChao/vicuna-7B-chemical", - "TinyPixel/llama2-7b-oa", - "chaoyi-wu/MedLLaMA_13B", - "edor/Platypus2-mini-7B", - "RoversX/llama-2-7b-hf-small-shards-Samantha-V1-SFT", - "venkycs/llama-v2-7b-32kC-Security", - "psyche/kollama2-7b", - "Fredithefish/Guanaco-7B-Uncensored", - "TheTravellingEngineer/llama2-7b-chat-hf-guanaco", - "ehartford/WizardLM-13B-Uncensored", - "PocketDoc/Dans-CreepingSenseOfDoom", - "wenge-research/yayi-7b-llama2", - "georgesung/llama2_7b_chat_uncensored", - "TinyPixel/llama2-7b-instruct", - "quantumaikr/QuantumLM-7B", - "xzuyn/MedicWizard-7B", - "wenge-research/yayi-7b-llama2", - "TinyPixel/lima-test", - "elyza/ELYZA-japanese-Llama-2-7b-instruct", - "lgaalves/llama-2-7b-hf_open-platypus", - "ziqingyang/chinese-alpaca-2-7b", - "TehVenom/Pygmalion-Vicuna-1.1-7b", - "meta-llama/Llama-2-7b-hf", - "bongchoi/test-llama2-7b", - "TaylorAI/Flash-Llama-7B", - "TheTravellingEngineer/llama2-7b-chat-hf-v2", - "TheTravellingEngineer/llama2-7b-chat-hf-v4", - "kashif/stack-llama-2", - "PeanutJar/LLaMa-2-PeanutButter_v18_A-7B", - "ToolBench/ToolLLaMA-7b-LoRA", - "Monero/WizardLM-13b-OpenAssistant-Uncensored", - "TheTravellingEngineer/llama2-7b-chat-hf-v2", - "TheTravellingEngineer/llama2-7b-chat-hf-v4", - "mrm8488/llama-2-coder-7b", - "elyza/ELYZA-japanese-Llama-2-7b-fast-instruct", - "clibrain/Llama-2-7b-ft-instruct-es", - "medalpaca/medalpaca-7b", - "TheBloke/tulu-7B-fp16", - "OpenBuddy/openbuddy-openllama-13b-v7-fp16", - "TaylorAI/FLAN-Llama-7B-2_Llama2-7B-Flash_868_full_model", - "Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload", - "jondurbin/airoboros-l2-7b-gpt4-2.0", - "dhmeltzer/llama-7b-SFT_ds_eli5_1024_r_64_alpha_16_merged", - "GOAT-AI/GOAT-7B-Community", - "AtomEchoAI/AtomGPT_56k", - "julianweng/Llama-2-7b-chat-orcah", - "TehVenom/Pygmalion-13b-Merged", - "jondurbin/airoboros-7b-gpt4-1.1", - "dhmeltzer/llama-7b-SFT_ds_wiki65k_1024_r_64_alpha_16_merged", - "bofenghuang/vigogne-7b-chat", - "lmsys/longchat-7b-v1.5-32k", - "jondurbin/airoboros-l2-7b-gpt4-m2.0", - "synapsoft/Llama-2-7b-chat-hf-flan2022-1.2M", - "jondurbin/airoboros-7b-gpt4-1.4", - "Charlie911/vicuna-7b-v1.5-lora-mctaco", - "yihan6324/instructmining-platypus-15k", - "meta-llama/Llama-2-7b-hf", - "TheTravellingEngineer/llama2-7b-chat-hf-v3", - "quantumaikr/KoreanLM-hf", - "openthaigpt/openthaigpt-1.0.0-alpha-7b-chat-ckpt-hf", - "TheBloke/Llama-2-7B-GPTQ", - "TheBloke/Llama-2-7B-GPTQ", - "LLMs/AlpacaGPT4-7B-elina", - "ehartford/Wizard-Vicuna-7B-Uncensored", - "TheBloke/Wizard-Vicuna-7B-Uncensored-HF", - "TheTravellingEngineer/llama2-7b-chat-hf-v3", - "golaxy/gowizardlm", - "ehartford/dolphin-llama2-7b", - "CHIH-HUNG/llama-2-7b-dolphin_10w-test", - "mncai/chatdoctor", - "psyche/kollama2-7b-v3", - "jondurbin/airoboros-7b-gpt4", - "jondurbin/airoboros-7b", - "TheBloke/airoboros-7b-gpt4-fp16", - "mosaicml/mpt-7b-8k-chat", - "elyza/ELYZA-japanese-Llama-2-7b", - "bofenghuang/vigogne-7b-instruct", - "jxhong/CAlign-alpaca-7b", - "golaxy/goims", - "jondurbin/airoboros-7b-gpt4-1.2", - "jphme/orca_mini_v2_ger_7b", - "psmathur/orca_mini_v2_7b", - "notstoic/PygmalionCoT-7b", - "golaxy/gogpt2-13b", - "golaxy/gogpt2-13b-chat", - "togethercomputer/LLaMA-2-7B-32K", - "TheBloke/wizardLM-7B-HF", - "keyfan/vicuna-chinese-replication-v1.1", - "golaxy/gogpt2-7b", - "aiplanet/effi-7b", - "arver/llama7b-qlora", - "titan087/OpenLlama13B-Guanaco", - "chavinlo/alpaca-native", - "project-baize/baize-healthcare-lora-7B", - "AlpinDale/pygmalion-instruct", - "openlm-research/open_llama_13b", - "jondurbin/airoboros-7b-gpt4-1.3", - "elyza/ELYZA-japanese-Llama-2-7b-fast", - "jondurbin/airoboros-gpt-3.5-turbo-100k-7b", - "uukuguy/speechless-codellama-orca-13b", - "bigcode/starcoderplus", - "TheBloke/guanaco-7B-HF", - "Neko-Institute-of-Science/metharme-7b", - "TigerResearch/tigerbot-7b-base", - "golaxy/gogpt-7b", - "togethercomputer/LLaMA-2-7B-32K", - "yhyhy3/open_llama_7b_v2_med_instruct", - "ajibawa-2023/carl-7b", - "stabilityai/stablelm-base-alpha-7b-v2", - "conceptofmind/LLongMA-2-7b-16k", - "TehVenom/Pygmalion_AlpacaLora-7b", - "jondurbin/airoboros-7b-gpt4-1.4.1-qlora", - "wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard", - "ausboss/llama7b-wizardlm-unfiltered", - "project-baize/baize-v2-7b", - "LMFlow/Robin-v2", - "HanningZhang/Robin-v2", - "LMFlow/Robin-7b-v2", - "OptimalScale/robin-7b-v2-delta", - "uukuguy/speechless-codellama-platypus-13b", - "jerryjalapeno/nart-100k-7b", - "wenge-research/yayi-13b-llama2", - "fireballoon/baichuan-vicuna-chinese-7b", - "jlevin/guanaco-unchained-llama-2-7b", - "csitfun/llama-7b-logicot", - "DevaMalla/llama7b_alpaca_1gpu_bf16", - "WeOpenML/PandaLM-Alpaca-7B-v1", - "illuin/test-custom-llama", - "yeontaek/WizardCoder-Python-13B-LoRa", - "ashercn97/giraffe-7b", - "mosaicml/mpt-7b-chat", - "abhishek/autotrain-llama-alpaca-peft-52508123785", - "Neko-Institute-of-Science/pygmalion-7b", - "TFLai/llama-7b-4bit-alpaca", - "huggingface/llama-7b", - "TheBloke/Planner-7B-fp16", - "shibing624/chinese-llama-plus-13b-hf", - "AGI-inc/lora_moe_7b_baseline", - "DevaMalla/llama-base-7b", - "AGI-inc/lora_moe_7b", - "togethercomputer/GPT-JT-6B-v0", - "ehartford/WizardLM-7B-Uncensored", - "shibing624/chinese-alpaca-plus-7b-hf", - "beomi/llama-2-ko-7b", - "mosaicml/mpt-7b-8k-instruct", - "Enno-Ai/ennodata-7b", - "mosaicml/mpt-7b-instruct", - "facebook/opt-iml-max-30b", - "WeOpenML/Alpaca-7B-v1", - "TheBloke/Project-Baize-v2-7B-GPTQ", - "codellama/CodeLlama-13b-Instruct-hf", - "TheBloke/CodeLlama-13B-Instruct-fp16", - "facebook/galactica-30b", - "FreedomIntelligence/phoenix-inst-chat-7b", - "openlm-research/open_llama_7b_v2", - "GeorgiaTechResearchInstitute/galpaca-30b", - "THUDM/chatglm2-6b", - "togethercomputer/GPT-JT-6B-v1", - "TheBloke/koala-7B-HF", - "nathan0/mpt_delta_tuned_model_v3", - "nathan0/mpt_delta_tuned_model_v2", - "GeorgiaTechResearchInstitute/galpaca-30b", - "JosephusCheung/Guanaco", - "shareAI/CodeLLaMA-chat-13b-Chinese", - "TigerResearch/tigerbot-7b-sft", - "Writer/InstructPalmyra-20b", - "OpenAssistant/codellama-13b-oasst-sft-v10", - "bigscience/bloomz-7b1-mt", - "nathan0/mpt_delta_tuned_model_v3", - "VMware/open-llama-7b-open-instruct", - "baichuan-inc/Baichuan-7B", - "anas-awadalla/mpt-7b", - "mosaicml/mpt-7b", - "bigscience/bloomz-7b1", - "ziqingyang/chinese-llama-2-7b", - "OpenAssistant/codellama-13b-oasst-sft-v10", - "wenge-research/yayi-7b", - "tiiuae/falcon-7b", - "togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1", - "togethercomputer/RedPajama-INCITE-7B-Instruct", - "TheBloke/landmark-attention-llama7b-fp16", - "togethercomputer/GPT-JT-Moderation-6B", - "h2oai/h2ogpt-gm-oasst1-en-1024-20b", - "dvruette/gpt-neox-20b-full-precision", - "TehVenom/Moderator-Chan_GPT-JT-6b", - "dvruette/oasst-gpt-neox-20b-1000-steps", - "AlekseyKorshuk/pygmalion-6b-vicuna-chatml", - "facebook/opt-66b", - "Salesforce/codegen-16B-nl", - "Vmware/open-llama-7b-v2-open-instruct", - "mosaicml/mpt-7b-storywriter", - "acrastt/Marx-3B-V2", - "openlm-research/open_llama_7b", - "Fredithefish/ReasonixPajama-3B-HF", - "togethercomputer/GPT-NeoXT-Chat-Base-20B", - "psmathur/orca_mini_13b", - "RWKV/rwkv-raven-14b", - "h2oai/h2ogpt-oasst1-512-20b", - "acrastt/Marx-3B", - "klosax/open_llama_13b_600bt_preview", - "synapsoft/Llama-2-7b-hf-flan2022-1.2M", - "OpenAssistant/oasst-sft-1-pythia-12b", - "golaxy/gogpt-7b-bloom", - "Writer/palmyra-large", - "psmathur/orca_mini_7b", - "dvruette/oasst-pythia-12b-6000-steps", - "NousResearch/CodeLlama-13b-hf", - "codellama/CodeLlama-13b-hf", - "h2oai/h2ogpt-gm-oasst1-multilang-1024-20b", - "VMware/open-llama-0.7T-7B-open-instruct-v1.1", - "dvruette/oasst-pythia-12b-flash-attn-5000-steps", - "dvruette/oasst-gpt-neox-20b-3000-steps", - "RobbeD/OpenLlama-Platypus-3B", - "facebook/opt-30b", - "acrastt/Puma-3B", - "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5", - "dvruette/oasst-pythia-12b-pretrained-sft", - "digitous/GPT-R", - "acrastt/Griffin-3B", - "togethercomputer/RedPajama-INCITE-Base-7B-v0.1", - "togethercomputer/RedPajama-INCITE-7B-Base", - "CobraMamba/mamba-gpt-3b-v3", - "Danielbrdz/CodeBarcenas-7b", - "l3utterfly/open-llama-3b-v2-layla", - "CobraMamba/mamba-gpt-3b-v2", - "OpenAssistant/pythia-12b-sft-v8-7k-steps", - "KoboldAI/GPT-NeoX-20B-Erebus", - "RobbeD/Orca-Platypus-3B", - "h2oai/h2ogpt-gm-oasst1-en-1024-12b", - "OpenAssistant/pythia-12b-sft-v8-2.5k-steps", - "AlekseyKorshuk/chatml-pyg-v1", - "togethercomputer/RedPajama-INCITE-Chat-7B-v0.1", - "togethercomputer/RedPajama-INCITE-7B-Chat", - "digitous/Javelin-R", - "dvruette/oasst-pythia-12b-reference", - "EleutherAI/gpt-neox-20b", - "KoboldAI/fairseq-dense-13B", - "OpenAssistant/pythia-12b-sft-v8-rlhf-2k-steps", - "codellama/CodeLlama-7b-Instruct-hf", - "digitous/Javelin-GPTJ", - "KoboldAI/GPT-NeoX-20B-Skein", - "digitous/Javalion-R", - "h2oai/h2ogpt-oasst1-512-12b", - "acrastt/Bean-3B", - "KoboldAI/GPT-J-6B-Skein", - "nomic-ai/gpt4all-j", - "databricks/dolly-v2-12b", - "TehVenom/Dolly_Shygmalion-6b-Dev_V8P2", - "databricks/dolly-v2-7b", - "Aspik101/WizardVicuna-Uncensored-3B-instruct-PL-lora_unload", - "digitous/Adventien-GPTJ", - "openlm-research/open_llama_3b_v2", - "RWKV/rwkv-4-14b-pile", - "Lazycuber/Janemalion-6B", - "OpenAssistant/pythia-12b-pre-v8-12.5k-steps", - "digitous/Janin-R", - "kfkas/Llama-2-ko-7b-Chat", - "heegyu/WizardVicuna-Uncensored-3B-0719", - "h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt", - "TaylorAI/Flash-Llama-3B", - "kfkas/Llama-2-ko-7b-Chat", - "digitous/Skegma-GPTJ", - "digitous/Javalion-GPTJ", - "Pirr/pythia-13b-deduped-green_devil", - "TehVenom/PPO_Shygmalion-V8p4_Dev-6b", - "dvruette/oasst-pythia-6.9b-4000-steps", - "heegyu/WizardVicuna-3B-0719", - "psmathur/orca_mini_3b", - "OpenAssistant/galactica-6.7b-finetuned", - "frank098/orca_mini_3b_juniper", - "PygmalionAI/pygmalion-6b", - "TehVenom/PPO_Pygway-V8p4_Dev-6b", - "TFLai/gpt-neox-20b-4bit-alpaca", - "Corianas/gpt-j-6B-Dolly", - "TehVenom/Dolly_Shygmalion-6b", - "digitous/Janin-GPTJ", - "TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4", - "EleutherAI/gpt-j-6b", - "KoboldAI/GPT-J-6B-Shinen", - "TehVenom/Dolly_Malion-6b", - "TehVenom/ChanMalion", - "Salesforce/codegen-6B-nl", - "Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4", - "KoboldAI/GPT-J-6B-Janeway", - "togethercomputer/RedPajama-INCITE-Chat-3B-v1", - "togethercomputer/Pythia-Chat-Base-7B", - "heegyu/RedTulu-Uncensored-3B-0719", - "KoboldAI/PPO_Pygway-6b-Mix", - "KoboldAI/OPT-13B-Erebus", - "KoboldAI/fairseq-dense-6.7B", - "EleutherAI/pythia-12b-deduped", - "pszemraj/pythia-6.9b-HC3", - "Fredithefish/Guanaco-3B-Uncensored-v2", - "facebook/opt-13b", - "TehVenom/GPT-J-Pyg_PPO-6B", - "EleutherAI/pythia-6.9b-deduped", - "Devio/test-1400", - "Fredithefish/Guanaco-3B-Uncensored", - "codellama/CodeLlama-7b-hf", - "acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1", - "Fredithefish/ScarletPajama-3B-HF", - "KoboldAI/OPT-13B-Nerybus-Mix", - "YeungNLP/firefly-bloom-7b1", - "DanielSc4/RedPajama-INCITE-Chat-3B-v1-RL-LoRA-8bit-test1", - "klosax/open_llama_7b_400bt_preview", - "KoboldAI/OPT-13B-Nerys-v2", - "TehVenom/PPO_Shygmalion-6b", - "amazon/LightGPT", - "KnutJaegersberg/black_goo_recipe_c", - "NousResearch/CodeLlama-7b-hf", - "togethercomputer/RedPajama-INCITE-Instruct-3B-v1", - "heegyu/WizardVicuna-open-llama-3b-v2", - "bigscience/bloom-7b1", - "Devio/test-22B", - "RWKV/rwkv-raven-7b", - "hakurei/instruct-12b", - "CobraMamba/mamba-gpt-3b", - "KnutJaegersberg/black_goo_recipe_a", - "acrastt/OmegLLaMA-3B", - "codellama/CodeLlama-7b-Instruct-hf", - "h2oai/h2ogpt-oig-oasst1-512-6_9b", - "KoboldAI/OPT-6.7B-Erebus", - "facebook/opt-6.7b", - "KnutJaegersberg/black_goo_recipe_d", - "KnutJaegersberg/LLongMA-3b-LIMA", - "KnutJaegersberg/black_goo_recipe_b", - "KoboldAI/OPT-6.7B-Nerybus-Mix", - "health360/Healix-3B", - "EleutherAI/pythia-12b", - "Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K", - "GeorgiaTechResearchInstitute/galactica-6.7b-evol-instruct-70k", - "h2oai/h2ogpt-oig-oasst1-256-6_9b", - "ikala/bloom-zh-3b-chat", - "Taekyoon/llama2-ko-7b-test", - "anhnv125/pygmalion-6b-roleplay", - "TehVenom/DiffMerge_Pygmalion_Main-onto-V8P4", - "KoboldAI/OPT-6B-nerys-v2", - "Lazycuber/pyg-instruct-wizardlm", - "Devio/testC", - "KoboldAI/OPT-30B-Erebus", - "Fredithefish/CrimsonPajama", - "togethercomputer/RedPajama-INCITE-Base-3B-v1", - "bigscience/bloomz-3b", - "conceptofmind/Open-LLongMA-3b", - "RWKV/rwkv-4-7b-pile", - "openlm-research/open_llama_3b", - "ewof/koishi-instruct-3b", - "DanielSc4/RedPajama-INCITE-Chat-3B-v1-FT-LoRA-8bit-test1", - "cerebras/Cerebras-GPT-13B", - "EleutherAI/pythia-6.7b", - "aisquared/chopt-2_7b", - "Azure99/blossom-v1-3b", - "PSanni/Deer-3b", - "bertin-project/bertin-gpt-j-6B-alpaca", - "OpenBuddy/openbuddy-openllama-3b-v10-bf16", - "KoboldAI/fairseq-dense-2.7B", - "ehartford/CodeLlama-34b-Instruct-hf", - "codellama/CodeLlama-34b-Instruct-hf", - "TheBloke/CodeLlama-34B-Instruct-fp16", - "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2", - "openlm-research/open_llama_7b_700bt_preview", - "NbAiLab/nb-gpt-j-6B-alpaca", - "KoboldAI/OPT-2.7B-Erebus", - "Writer/camel-5b-hf", - "EleutherAI/pythia-2.7b", - "facebook/xglm-7.5B", - "EleutherAI/pythia-2.8b-deduped", - "klosax/open_llama_3b_350bt_preview", - "klosax/openllama-3b-350bt", - "KoboldAI/OPT-2.7B-Nerybus-Mix", - "KoboldAI/GPT-J-6B-Adventure", - "cerebras/Cerebras-GPT-6.7B", - "TFLai/pythia-2.8b-4bit-alpaca", - "facebook/opt-2.7b", - "KoboldAI/OPT-2.7B-Nerys-v2", - "bigscience/bloom-3b", - "Devio/test100", - "RWKV/rwkv-raven-3b", - "Azure99/blossom-v2-3b", - "codellama/CodeLlama-34b-Python-hf", - "bhenrym14/airoboros-33b-gpt4-1.4.1-PI-8192-fp16", - "EleutherAI/gpt-neo-2.7B", - "danielhanchen/open_llama_3b_600bt_preview", - "open-llm-leaderboard/starchat-alpha", - "pythainlp/wangchanglm-7.5B-sft-en-sharded", - "beaugogh/pythia-1.4b-deduped-sharegpt", - "HWERI/pythia-1.4b-deduped-sharegpt", - "OpenAssistant/stablelm-7b-sft-v7-epoch-3", - "codellama/CodeLlama-7b-Python-hf", - "aisquared/chopt-1_3b", - "PygmalionAI/metharme-1.3b", - "Linly-AI/Chinese-LLaMA-2-13B-hf", - "chargoddard/llama-2-34b-uncode", - "RWKV/rwkv-4-3b-pile", - "pythainlp/wangchanglm-7.5B-sft-enth", - "MBZUAI/LaMini-GPT-1.5B", - "Writer/palmyra-base", - "KoboldAI/fairseq-dense-1.3B", - "EleutherAI/pythia-1.4b-deduped", - "MBZUAI/lamini-neo-1.3b", - "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt", - "sartmis1/starcoder-finetune-openapi", - "MayaPH/opt-flan-iml-6.7b", - "facebook/xglm-4.5B", - "WizardLM/WizardCoder-15B-V1.0", - "facebook/opt-iml-max-1.3b", - "stabilityai/stablelm-tuned-alpha-7b", - "aisquared/dlite-v2-1_5b", - "stabilityai/stablelm-base-alpha-7b", - "sartmis1/starcoder-finetune-selfinstruct", - "lizhuang144/starcoder_mirror", - "bigcode/starcoder", - "TheBloke/CodeLlama-34B-Python-fp16", - "open-llm-leaderboard/bloomz-1b7-4bit-alpaca-auto-eval-adapter-applied", - "ehartford/CodeLlama-34b-Python-hf", - "codellama/CodeLlama-7b-Python-hf", - "GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct", - "LoupGarou/WizardCoder-Guanaco-15B-V1.0", - "golaxy/gogpt-3b-bloom", - "EleutherAI/pythia-1.3b", - "codellama/CodeLlama-13b-Python-hf", - "hakurei/lotus-12B", - "NYTK/PULI-GPTrio", - "facebook/opt-1.3b", - "TheBloke/CodeLlama-13B-Python-fp16", - "codellama/CodeLlama-13b-Python-hf", - "RWKV/rwkv-raven-1b5", - "PygmalionAI/pygmalion-2.7b", - "bigscience/bloom-1b7", - "gpt2-xl", - "LoupGarou/WizardCoder-Guanaco-15B-V1.1", - "RWKV/rwkv-4-1b5-pile", - "codellama/CodeLlama-34b-hf", - "NousResearch/CodeLlama-34b-hf", - "rinna/bilingual-gpt-neox-4b-8k", - "lxe/Cerebras-GPT-2.7B-Alpaca-SP", - "cerebras/Cerebras-GPT-2.7B", - "jzjiao/opt-1.3b-rlhf", - "EleutherAI/gpt-neo-1.3B", - "aisquared/dlite-v1-1_5b", - "Corianas/Quokka_2.7b", - "MrNJK/gpt2-xl-sft", - "facebook/galactica-1.3b", - "aisquared/dlite-v2-774m", - "EleutherAI/pythia-1b-deduped", - "Kunhao/pile-7b-250b-tokens", - "w601sxs/b1ade-1b", - "rinna/bilingual-gpt-neox-4b", - "shaohang/SparseOPT-1.3B", - "shaohang/Sparse0.5_OPT-1.3", - "EleutherAI/polyglot-ko-12.8b", - "Salesforce/codegen-6B-multi", - "bigscience/bloom-1b1", - "TFLai/gpt-neo-1.3B-4bit-alpaca", - "FabbriSimo01/Bloom_1b_Quantized", - "MBZUAI/LaMini-GPT-774M", - "Locutusque/gpt2-large-conversational", - "Devio/test-3b", - "stabilityai/stablelm-tuned-alpha-3b", - "PygmalionAI/pygmalion-1.3b", - "KoboldAI/fairseq-dense-355M", - "Rachneet/gpt2-xl-alpaca", - "gpt2-large", - "Mikivis/gpt2-large-lora-sft", - "stabilityai/stablelm-base-alpha-3b", - "gpt2-medium", - "Kunhao/pile-7b", - "aisquared/dlite-v1-774m", - "aisquared/dlite-v2-355m", - "YeungNLP/firefly-bloom-2b6-v2", - "KnutJaegersberg/gpt-2-xl-EvolInstruct", - "KnutJaegersberg/galactica-orca-wizardlm-1.3b", - "cerebras/Cerebras-GPT-1.3B", - "FabbriSimo01/Cerebras_1.3b_Quantized", - "facebook/xglm-1.7B", - "EleutherAI/pythia-410m-deduped", - "TheBloke/GPlatty-30B-SuperHOT-8K-fp16", - "DataLinguistic/DataLinguistic-34B-V1.0", - "Corianas/Quokka_1.3b", - "TheTravellingEngineer/bloom-560m-RLHF-v2", - "Corianas/1.3b", - "RWKV/rwkv-4-430m-pile", - "porkorbeef/Llama-2-13b-sf", - "xhyi/PT_GPTNEO350_ATG", - "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ", - "bigscience/bloomz-560m", - "TheBloke/medalpaca-13B-GPTQ-4bit", - "TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16", - "aisquared/dlite-v1-355m", - "uukuguy/speechless-codellama-orca-airoboros-13b-0.10e", - "yhyhy3/med-orca-instruct-33b", - "TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16", - "TheTravellingEngineer/bloom-1b1-RLHF", - "MBZUAI/lamini-cerebras-1.3b", - "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1", - "TheBloke/WizardLM-7B-uncensored-GPTQ", - "TheBloke/EverythingLM-13B-16K-GPTQ", - "quantumaikr/open_llama_7b_hf", - "TheBloke/chronos-wizardlm-uc-scot-st-13B-GPTQ", - "TheBloke/WizardLM-30B-Uncensored-GPTQ", - "IDEA-CCNL/Ziya-LLaMA-13B-v1", - "Phind/Phind-CodeLlama-34B-v1", - "robowaifudev/megatron-gpt2-345m", - "MayaPH/GodziLLa-30B-instruct", - "TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16", - "uukuguy/speechless-codellama-orca-platypus-13b-0.10e", - "doas/test2", - "BreadAi/PM_modelV2", - "bigcode/santacoder", - "TheBloke/wizard-vicuna-13B-GPTQ", - "porkorbeef/Llama-2-13b", - "TehVenom/DiffMerge-DollyGPT-Pygmalion", - "PygmalionAI/pygmalion-350m", - "TheBloke/orca_mini_v3_7B-GPTQ", - "TheBloke/WizardLM-Uncensored-SuperCOT-StoryTelling-30B-GPTQ", - "TheBloke/WizardLM-30B-GPTQ", - "bigscience/bloom-560m", - "TFLai/gpt2-turkish-uncased", - "TheBloke/guanaco-33B-GPTQ", - "TheBloke/openchat_v2_openorca_preview-GPTQ", - "porkorbeef/Llama-2-13b-public", - "TheBloke/LongChat-13B-GPTQ", - "yhyhy3/med-orca-instruct-33b", - "TheBloke/airoboros-33B-gpt4-1-4-SuperHOT-8K-fp16", - "TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16", - "MayaPH/FinOPT-Franklin", - "TheBloke/WizardLM-33B-V1.0-Uncensored-GPTQ", - "TheBloke/Project-Baize-v2-13B-GPTQ", - "malhajar/Platypus2-70B-instruct-4bit-gptq", - "KoboldAI/OPT-350M-Erebus", - "rishiraj/bloom-560m-guanaco", - "Panchovix/WizardLM-33B-V1.0-Uncensored-SuperHOT-8k", - "doas/test5", - "vicgalle/alpaca-7b", - "beomi/KoAlpaca-Polyglot-5.8B", - "Phind/Phind-CodeLlama-34B-Python-v1", - "timdettmers/guanaco-65b-merged", - "TheBloke/wizard-mega-13B-GPTQ", - "MayaPH/GodziLLa-30B-plus", - "TheBloke/Platypus-30B-SuperHOT-8K-fp16", - "facebook/opt-350m", - "KoboldAI/OPT-350M-Nerys-v2", - "TheBloke/robin-33B-v2-GPTQ", - "jaspercatapang/Echidna-30B", - "TheBloke/llama-30b-supercot-SuperHOT-8K-fp16", - "marcchew/test1", - "Harshvir/LaMini-Neo-1.3B-Mental-Health_lora", - "golaxy/gogpt-560m", - "TheBloke/orca_mini_13B-GPTQ", - "Panchovix/airoboros-33b-gpt4-1.2-SuperHOT-8k", - "Aspik101/tulu-7b-instruct-pl-lora_unload", - "Phind/Phind-CodeLlama-34B-v2", - "BreadAi/MusePy-1-2", - "cerebras/Cerebras-GPT-590M", - "microsoft/CodeGPT-small-py", - "victor123/WizardLM-13B-1.0", - "OptimalScale/robin-65b-v2-delta", - "voidful/changpt-bart", - "FabbriSimo01/GPT_Large_Quantized", - "MayaPH/FinOPT-Lincoln", - "KoboldAI/fairseq-dense-125M", - "SebastianSchramm/Cerebras-GPT-111M-instruction", - "TheTravellingEngineer/bloom-560m-RLHF", - "breadlicker45/dough-instruct-base-001", - "WizardLM/WizardLM-30B-V1.0", - "WizardLM/WizardLM-30B-V1.0", - "WizardLM/WizardLM-30B-V1.0", - "TaylorAI/Flash-Llama-30M-20001", - "porkorbeef/Llama-2-13b-12_153950", - "huggingtweets/bladeecity-jerma985", - "KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct", - "bhenrym14/airoboros-33b-gpt4-1.4.1-lxctx-PI-16384-fp16", - "microsoft/DialoGPT-small", - "Corianas/590m", - "facebook/xglm-564M", - "EleutherAI/gpt-neo-125m", - "EleutherAI/pythia-160m-deduped", - "klosax/pythia-160m-deduped-step92k-193bt", - "MBZUAI/lamini-neo-125m", - "bigcode/tiny_starcoder_py", - "concedo/OPT-19M-ChatSalad", - "anton-l/gpt-j-tiny-random", - "grantprice/Cerebras-GPT-590M-finetuned-DND", - "deepnight-research/zsc-text", - "WangZeJun/bloom-820m-chat", - "cerebras/Cerebras-GPT-256M", - "ai-forever/rugpt3large_based_on_gpt2", - "alibidaran/medical_transcription_generator", - "Deci/DeciCoder-1b", - "microsoft/DialoGPT-medium", - "ogimgio/gpt-neo-125m-neurallinguisticpioneers", - "open-llm-leaderboard/bloom-560m-4bit-alpaca-auto-eval-adapter-applied", - "BreadAi/gpt-YA-1-1_160M", - "microsoft/DialoGPT-large", - "facebook/opt-125m", - "huggingtweets/jerma985", - "Locutusque/gpt2-conversational-or-qa", - "concedo/Pythia-70M-ChatSalad", - "roneneldan/TinyStories-1M", - "BreadAi/DiscordPy", - "bigcode/gpt_bigcode-santacoder", - "Tincando/fiction_story_generator", - "klosax/pythia-70m-deduped-step44k-92bt", - "Quake24/easyTermsSummerizer", - "BreadAi/gpt-YA-1-1_70M", - "EleutherAI/pythia-160m", - "euclaise/gpt-neox-122m-minipile-digits", - "MBZUAI/lamini-cerebras-590m", - "nicholasKluge/Aira-124M", - "MayaPH/FinOPT-Washington", - "cyberagent/open-calm-large", - "BreadAi/StoryPy", - "EleutherAI/pythia-70m", - "BreadAi/gpt-Youtube", - "roneneldan/TinyStories-33M", - "EleutherAI/pythia-70m-deduped", - "lgaalves/gpt2_guanaco-dolly-platypus", - "Corianas/Quokka_590m", - "lgaalves/gpt2_platypus-dolly-guanaco", - "cyberagent/open-calm-7b", - "RWKV/rwkv-4-169m-pile", - "gpt2", - "roneneldan/TinyStories-28M", - "lgaalves/gpt2_open-platypus", - "gpt2", - "SaylorTwift/gpt2_test", - "roneneldan/TinyStories-3M", - "nthngdy/pythia-owt2-70m-50k", - "Corianas/256_5epoch", - "roneneldan/TinyStories-8M", - "lgaalves/gpt2-dolly", - "nthngdy/pythia-owt2-70m-100k", - "aisquared/dlite-v2-124m", - "mncai/SGPT-1.3B-insurance-epoch10", - "huggingtweets/gladosystem", - "abhiramtirumala/DialoGPT-sarcastic-medium", - "MBZUAI/lamini-cerebras-256m", - "cerebras/Cerebras-GPT-111M", - "uberkie/metharme-1.3b-finetuned", - "MBZUAI/lamini-cerebras-111m", - "psyche/kogpt", - "Corianas/Quokka_256m", - "vicgalle/gpt2-alpaca-gpt4", - "aisquared/dlite-v1-124m", - "Mikivis/xuanxuan", - "MBZUAI/LaMini-GPT-124M", - "vicgalle/gpt2-alpaca", - "huashiyiqike/testmodel", - "Corianas/111m", - "baseline", -] diff --git a/src/voting/vote_system.py b/src/voting/vote_system.py deleted file mode 100644 index 0ab75738159cc377a480c566393f872d615da898..0000000000000000000000000000000000000000 --- a/src/voting/vote_system.py +++ /dev/null @@ -1,155 +0,0 @@ -import json -import logging -import pathlib -import pandas as pd -import gradio as gr -import schedule -import time -from datetime import datetime, timezone -from src.display.utils import EvalQueueColumn - -from src.envs import API - -# Set up logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -class VoteManager: - def __init__(self, votes_path, eval_requests_path, repo_id): - self.votes_path = votes_path - self.eval_requests_path = eval_requests_path - self.repo_id = repo_id - self.vote_dataset = self.read_vote_dataset() - self.vote_check_set = self.make_check_set(self.vote_dataset) - self.votes_to_upload = [] - - def init_vote_dataset(self): - self.vote_dataset = self.read_vote_dataset() - self.vote_check_set = self.make_check_set(self.vote_dataset) - - def read_vote_dataset(self): - result = [] - votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl" - if votes_file.exists(): - with open(votes_file, "r") as f: - for line in f: - data = json.loads(line.strip()) - result.append(data) - result = pd.DataFrame(result) - return result - - def make_check_set(self, vote_dataset: pd.DataFrame): - result = list() - for row in vote_dataset.itertuples(index=False, name='vote'): - result.append((row.model, row.revision, row.username)) - return set(result) - - def get_model_revision(self, selected_model: str) -> str: - """Fetch the revision for the given model from the request files.""" - for user_folder in pathlib.Path(self.eval_requests_path).iterdir(): - if user_folder.is_dir(): - for file in user_folder.glob("*.json"): - with open(file, "r") as f: - data = json.load(f) - if data.get("model") == selected_model: - return data.get("revision", "main") - return "main" - - def create_request_vote_df(self, pending_models_df: gr.Dataframe): - if pending_models_df.empty or "model_name" not in pending_models_df.columns: - return pending_models_df - - self.vote_dataset = self.read_vote_dataset() - vote_counts = self.vote_dataset.groupby(['model', 'revision']).size().reset_index(name='vote_count') - - pending_models_df_votes = pd.merge( - pending_models_df, - vote_counts, - left_on=["model_name", 'revision'], - right_on=['model', 'revision'], - how='left' - ) - # Filling empty votes - pending_models_df_votes['vote_count'] = pending_models_df_votes['vote_count'].fillna(0) - pending_models_df_votes = pending_models_df_votes.sort_values(by=["vote_count", "model_name"], ascending=[False, True]) - # Removing useless columns - pending_models_df_votes = pending_models_df_votes.drop(["model_name", "model"], axis=1) - return pending_models_df_votes - - # Function to be called when a user votes for a model - def add_vote( - self, - selected_model: str, - pending_models_df: gr.Dataframe | None, - profile: gr.OAuthProfile | None - ): - logger.debug(f"Type of list before usage: {type(list)}") - # model_name, revision, user_id, timestamp - if selected_model in ["str", ""]: - gr.Warning("No model selected") - return - - if profile is None: - gr.Warning("Hub Login required") - return - - vote_username = profile.username - model_revision = self.get_model_revision(selected_model) - - # tuple (immutable) for checking than already voted for model - check_tuple = (selected_model, model_revision, vote_username) - if check_tuple in self.vote_check_set: - gr.Warning("Already voted for this model") - return - - current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - vote_obj = { - "model": selected_model, - "revision": model_revision, - "username": vote_username, - "timestamp": current_time - } - - # Append the vote to the JSONL file - try: - votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl" - with open(votes_file, "a") as f: - f.write(json.dumps(vote_obj) + "\n") - logger.info(f"Vote added locally: {vote_obj}") - - self.votes_to_upload.append(vote_obj) - except Exception as e: - logger.error(f"Failed to write vote to file: {e}") - gr.Warning("Failed to record vote. Please try again") - return - - self.vote_check_set.add(check_tuple) - gr.Info(f"Voted for {selected_model}") - - if pending_models_df is None: - return - - return self.create_request_vote_df(pending_models_df) - - def upload_votes(self): - if self.votes_to_upload: - votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl" - try: - with open(votes_file, "rb") as f: - API.upload_file( - path_or_fileobj=f, - path_in_repo="votes_data.jsonl", - repo_id=self.repo_id, - repo_type="dataset", - commit_message="Updating votes_data.jsonl with new votes", - ) - logger.info("Votes uploaded to votes repository") - self.votes_to_upload.clear() - except Exception as e: - logger.error(f"Failed to upload votes to repository: {e}") - -def run_scheduler(vote_manager): - while True: - schedule.run_pending() - time.sleep(1) diff --git a/tests/submission/test_user_submission_permission.py b/tests/submission/test_user_submission_permission.py deleted file mode 100644 index 2f459f7b7b86282a922f6c22e4f6a79daec1bcde..0000000000000000000000000000000000000000 --- a/tests/submission/test_user_submission_permission.py +++ /dev/null @@ -1,98 +0,0 @@ -import unittest -from unittest.mock import patch -from datetime import datetime, timedelta, timezone - -from src.submission.check_validity import user_submission_permission -from src.envs import RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA - -class TestUserSubmissionPermission(unittest.TestCase): - - def setUp(self): - self.user_name = "test_user" - self.rate_limit_period = RATE_LIMIT_PERIOD - self.rate_limit_quota = RATE_LIMIT_QUOTA - self.fixed_now = datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc) - # Submission dates that simulate various test cases - self.users_to_submission_dates = { - "test_user": [ - (self.fixed_now - timedelta(days=1)).isoformat(), - (self.fixed_now - timedelta(days=2)).isoformat(), - (self.fixed_now - timedelta(days=3)).isoformat(), - (self.fixed_now - timedelta(days=4)).isoformat(), - ] - } - - @staticmethod - def fixed_datetime_now(tz=None): - return datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc) - - @patch('src.submission.check_validity.datetime') - def test_user_below_quota(self, mock_datetime): - mock_datetime.now.side_effect = self.fixed_datetime_now - mock_datetime.fromisoformat = datetime.fromisoformat - allowed, message = user_submission_permission( - self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota - ) - self.assertTrue(allowed) - - @patch('src.submission.check_validity.datetime') - def test_user_at_quota(self, mock_datetime): - mock_datetime.now.side_effect = self.fixed_datetime_now - mock_datetime.fromisoformat = datetime.fromisoformat - - # Add one more submission to reach the quota - self.users_to_submission_dates["test_user"].append(self.fixed_now.isoformat()) - - allowed, message = user_submission_permission( - self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota - ) - self.assertFalse(allowed) - expected_message = ( - f"Organisation or user `{self.user_name}` already has {self.rate_limit_quota} model requests submitted " - f"in the last {self.rate_limit_period} days.\n" - "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗" - ) - self.assertEqual(message, expected_message) - - @patch('src.submission.check_validity.datetime') - def test_user_above_quota(self, mock_datetime): - mock_datetime.now.side_effect = self.fixed_datetime_now - mock_datetime.fromisoformat = datetime.fromisoformat - # Add more than quota submissions - for _ in range(self.rate_limit_quota + 1): - self.users_to_submission_dates["test_user"].append(self.fixed_now.isoformat()) - allowed, message = user_submission_permission( - self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota - ) - self.assertFalse(allowed) - - def test_user_no_previous_submissions(self): - allowed, message = user_submission_permission( - "new_user", self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota - ) - self.assertTrue(allowed) - - @patch('src.submission.check_validity.HAS_HIGHER_RATE_LIMIT', ["specific_user"]) - @patch('src.submission.check_validity.datetime') - def test_user_higher_rate_limit(self, mock_datetime): - mock_datetime.now.side_effect = self.fixed_datetime_now - mock_datetime.fromisoformat = datetime.fromisoformat - self.users_to_submission_dates["specific_user"] = [self.fixed_now.isoformat()] * (self.rate_limit_quota + 1) - allowed, message = user_submission_permission( - "specific_user", self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota - ) - self.assertTrue(allowed) - - @patch('src.submission.check_validity.datetime') - def test_submission_just_outside_window(self, mock_datetime): - mock_datetime.now.side_effect = self.fixed_datetime_now - mock_datetime.fromisoformat = datetime.fromisoformat - old_submission = (self.fixed_now - timedelta(days=self.rate_limit_period, seconds=1)).isoformat() - self.users_to_submission_dates["test_user"] = [old_submission] - allowed, message = user_submission_permission( - self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota - ) - self.assertTrue(allowed) - -if __name__ == '__main__': - unittest.main()