
## acge model

acge是一个通用的文本编码模型，主要有以下模型：

|     Model Name     | Model Size (GB) | Dimension | Sequence Length | Language | Need instruction for retrieval? |
|:------------------:|:---------------:|:---------:|:---------------:|:--------:|:-------------------------------:|
|  acge-large-zh   |      0.65       |   1024    |      1024       | Chinese  |               NO               |
|   acge-base-zh   |       To be continuted       |       |            | Chinese  |               No               |


## Metric

#### C-MTEB leaderboard (Chinese)

|     Model Name     | Model Size (GB) | Dimension | Sequence Length | Average (35) | Classification (9) | Clustering (4) | Pair Classification (2) | Reranking (4) | Retrieval (8) | STS (8) |
|:------------------:|:---------------:|:---------:|:---------------:|:------------:|:------------------:|:--------------:|:-----------------------:|:-------------:|:-------------:|:-------:|
|    acge-large-zh   |      0.65       |   1024    |      1024       |    67.0      |       73.39        |     55.89      |          81.38          |     66.59     |     70.96     |  58.02  |

#### Reproduce our results

**C-MTEB:** 

```python
import torch
import argparse
import functools
from C_MTEB.tasks import *
from typing import List, Dict
from sentence_transformers import SentenceTransformer
from mteb import MTEB, DRESModel


class RetrievalModel(DRESModel):
    def __init__(self, encoder, **kwargs):
        self.encoder = encoder

    def encode_queries(self, queries: List[str], **kwargs) -> np.ndarray:
        input_texts = ['{}'.format(q) for q in queries]
        return self._do_encode(input_texts)

    def encode_corpus(self, corpus: List[Dict[str, str]], **kwargs) -> np.ndarray:
        input_texts = ['{} {}'.format(doc.get('title', ''), doc['text']).strip() for doc in corpus]
        input_texts = ['{}'.format(t) for t in input_texts]
        return self._do_encode(input_texts)

    @torch.no_grad()
    def _do_encode(self, input_texts: List[str]) -> np.ndarray:
        return self.encoder.encode(
            sentences=input_texts,
            batch_size=512,
            normalize_embeddings=True,
            convert_to_numpy=True
        )


def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--model_name_or_path', default="acge-large-zh", type=str)
    parser.add_argument('--task_type', default=None, type=str)
    parser.add_argument('--pooling_method', default='cls', type=str)
    parser.add_argument('--output_dir', default='zh_results',
                        type=str, help='output directory')
    parser.add_argument('--max_len', default=1024, type=int, help='max length')
    return parser.parse_args()


if __name__ == '__main__':
    args = get_args()
    encoder = SentenceTransformer(args.model_name_or_path).half()
    encoder.encode = functools.partial(encoder.encode, normalize_embeddings=True)
    encoder.max_seq_length = int(args.max_len)

    task_names = [t.description["name"] for t in MTEB(task_types=args.task_type,
                                                      task_langs=['zh', 'zh-CN']).tasks]
    TASKS_WITH_PROMPTS = ["T2Retrieval", "MMarcoRetrieval", "DuRetrieval", "CovidRetrieval", "CmedqaRetrieval",
                          "EcomRetrieval", "MedicalRetrieval", "VideoRetrieval"]
    for task in task_names:
        evaluation = MTEB(tasks=[task], task_langs=['zh', 'zh-CN'])
        if task in TASKS_WITH_PROMPTS:
            evaluation.run(RetrievalModel(encoder), output_folder=args.output_dir, overwrite_results=False)
        else:
            evaluation.run(encoder, output_folder=args.output_dir, overwrite_results=False)


```


## Usage

#### acge 中文系列模型

在sentence-transformer库中的使用方法：

```python
from sentence_transformers import SentenceTransformer

sentences = ["数据1", "数据2"]
model = SentenceTransformer('acge-large-zh')
print(model.max_seq_length)
embeddings_1 = model.encode(sentences, normalize_embeddings=True)
embeddings_2 = model.encode(sentences, normalize_embeddings=True)
similarity = embeddings_1 @ embeddings_2.T
print(similarity)
```

直接使用transformers库：

```python
from transformers import AutoModel, AutoTokenizer
from sklearn.preprocessing import normalize

model = AutoModel.from_pretrained('acge-large-zh')
tokenizer = AutoTokenizer.from_pretrained('acge-large-zh')
sentences = ["数据1", "数据ABCDEFGH"]
batch_data = tokenizer(
    batch_text_or_text_pairs=sentences,
    padding="longest",
    return_tensors="pt",
    max_length=1024,
    truncation=True,
)
attention_mask = batch_data["attention_mask"]
model_output = model(**batch_data)
last_hidden = model_output.last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0)
vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
vectors = normalize(vectors, norm="l2", axis=1, )
print(vectors.shape)  # 2,768
```

#### acge models for English

**Using Sentence-Transformers:**

```python
from sentence_transformers import SentenceTransformer

sentences = ["one car come", "one car go"]
model = SentenceTransformer('acge-large-zh')
print(model.max_seq_length)
embeddings_1 = model.encode(sentences, normalize_embeddings=True)
embeddings_2 = model.encode(sentences, normalize_embeddings=True)
similarity = embeddings_1 @ embeddings_2.T
print(similarity)
```

**Using HuggingFace Transformers:**

```python
from transformers import AutoModel, AutoTokenizer
from sklearn.preprocessing import normalize

model = AutoModel.from_pretrained('acge-large-zh')
tokenizer = AutoTokenizer.from_pretrained('acge-large-zh')
sentences = ["one car come", "one car go"]
batch_data = tokenizer(
    batch_text_or_text_pairs=sentences,
    padding="longest",
    return_tensors="pt",
    max_length=1024,
    truncation=True,
)
attention_mask = batch_data["attention_mask"]
model_output = model(**batch_data)
last_hidden = model_output.last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0)
vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
vectors = normalize(vectors, norm="l2", axis=1, )
print(vectors.shape)  # 2,768
```


{"tags": ["mteb", "sentence-transformers", "feature-extraction", "sentence-similarity"], "pipeline_tag": "sentence-similarity", "model-index": [{"name": "acge_large_zh", "results": [{"task": {"type": "STS"}, "dataset": {"name": "MTEB AFQMC", "type": "C-MTEB/AFQMC", "config": "default", "split": "validation", "revision": "None"}, "metrics": [{"type": "cos_sim_pearson", "value": 49.886787153993325}, {"type": "cos_sim_spearman", "value": 55.171124956893756}, {"type": "euclidean_pearson", "value": 53.335095346764646}, {"type": "euclidean_spearman", "value": 55.17112495483507}, {"type": "manhattan_pearson", "value": 53.33166516266124}, {"type": "manhattan_spearman", "value": 55.17617113643239}]}, {"task": {"type": "STS"}, "dataset": {"name": "MTEB ATEC", "type": "C-MTEB/ATEC", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "cos_sim_pearson", "value": 49.146852231760285}, {"type": "cos_sim_spearman", "value": 55.59605872485787}, {"type": "euclidean_pearson", "value": 56.15427063073917}, {"type": "euclidean_spearman", "value": 55.59605534700307}, {"type": "manhattan_pearson", "value": 56.14142018120689}, {"type": "manhattan_spearman", "value": 55.60169369486765}]}, {"task": {"type": "Classification"}, "dataset": {"name": "MTEB AmazonReviewsClassification (zh)", "type": "mteb/amazon_reviews_multi", "config": "zh", "split": "test", "revision": "1399c76144fd37290681b995c656ef9b2e06e26d"}, "metrics": [{"type": "accuracy", "value": 50.304}, {"type": "f1", "value": 49.19173527843529}]}, {"task": {"type": "STS"}, "dataset": {"name": "MTEB BQ", "type": "C-MTEB/BQ", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "cos_sim_pearson", "value": 63.48342875419366}, {"type": "cos_sim_spearman", "value": 65.33453641948725}, {"type": "euclidean_pearson", "value": 63.982308524093746}, {"type": "euclidean_spearman", "value": 65.33454681198816}, {"type": "manhattan_pearson", "value": 63.934639741619236}, {"type": "manhattan_spearman", "value": 65.28721332786898}]}, {"task": {"type": "Clustering"}, "dataset": {"name": "MTEB CLSClusteringP2P", "type": "C-MTEB/CLSClusteringP2P", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "v_measure", "value": 43.556554499458485}]}, {"task": {"type": "Clustering"}, "dataset": {"name": "MTEB CLSClusteringS2S", "type": "C-MTEB/CLSClusteringS2S", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "v_measure", "value": 41.066304244195074}]}, {"task": {"type": "Reranking"}, "dataset": {"name": "MTEB CMedQAv1", "type": "C-MTEB/CMedQAv1-reranking", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "map", "value": 86.48231157026598}, {"type": "mrr", "value": 89.09583333333333}]}, {"task": {"type": "Reranking"}, "dataset": {"name": "MTEB CMedQAv2", "type": "C-MTEB/CMedQAv2-reranking", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "map", "value": 87.28443790630314}, {"type": "mrr", "value": 89.9634126984127}]}, {"task": {"type": "Retrieval"}, "dataset": {"name": "MTEB CmedqaRetrieval", "type": "C-MTEB/CmedqaRetrieval", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map_at_1", "value": 24.807000000000002}, {"type": "map_at_10", "value": 37.037}, {"type": "map_at_100", "value": 38.829}, {"type": "map_at_1000", "value": 38.95}, {"type": "map_at_3", "value": 32.927}, {"type": "map_at_5", "value": 35.150999999999996}, {"type": "mrr_at_1", "value": 37.834}, {"type": "mrr_at_10", "value": 46.022}, {"type": "mrr_at_100", "value": 46.977000000000004}, {"type": "mrr_at_1000", "value": 47.024}, {"type": "mrr_at_3", "value": 43.498}, {"type": "mrr_at_5", "value": 44.914}, {"type": "ndcg_at_1", "value": 37.834}, {"type": "ndcg_at_10", "value": 43.689}, {"type": "ndcg_at_100", "value": 50.786}, {"type": "ndcg_at_1000", "value": 52.871}, {"type": "ndcg_at_3", "value": 38.412}, {"type": "ndcg_at_5", "value": 40.499}, {"type": "precision_at_1", "value": 37.834}, {"type": "precision_at_10", "value": 9.685}, {"type": "precision_at_100", "value": 1.552}, {"type": "precision_at_1000", "value": 0.182}, {"type": "precision_at_3", "value": 21.639}, {"type": "precision_at_5", "value": 15.679000000000002}, {"type": "recall_at_1", "value": 24.807000000000002}, {"type": "recall_at_10", "value": 54.474999999999994}, {"type": "recall_at_100", "value": 83.864}, {"type": "recall_at_1000", "value": 97.67099999999999}, {"type": "recall_at_3", "value": 38.562999999999995}, {"type": "recall_at_5", "value": 45.07}]}, {"task": {"type": "PairClassification"}, "dataset": {"name": "MTEB Cmnli", "type": "C-MTEB/CMNLI", "config": "default", "split": "validation", "revision": "None"}, "metrics": [{"type": "cos_sim_accuracy", "value": 80.0}, {"type": "cos_sim_ap", "value": 87.29832373329583}, {"type": "cos_sim_f1", "value": 81.44535579694082}, {"type": "cos_sim_precision", "value": 77.42887249736566}, {"type": "cos_sim_recall", "value": 85.90133270984335}, {"type": "dot_accuracy", "value": 80.0}, {"type": "dot_ap", "value": 87.29560107851526}, {"type": "dot_f1", "value": 81.44535579694082}, {"type": "dot_precision", "value": 77.42887249736566}, {"type": "dot_recall", "value": 85.90133270984335}, {"type": "euclidean_accuracy", "value": 80.0}, {"type": "euclidean_ap", "value": 87.29832836070828}, {"type": "euclidean_f1", "value": 81.44535579694082}, {"type": "euclidean_precision", "value": 77.42887249736566}, {"type": "euclidean_recall", "value": 85.90133270984335}, {"type": "manhattan_accuracy", "value": 79.9158147925436}, {"type": "manhattan_ap", "value": 87.30770912752833}, {"type": "manhattan_f1", "value": 81.46225394765305}, {"type": "manhattan_precision", "value": 75.78989736365466}, {"type": "manhattan_recall", "value": 88.05237315875614}, {"type": "max_accuracy", "value": 80.0}, {"type": "max_ap", "value": 87.30770912752833}, {"type": "max_f1", "value": 81.46225394765305}]}, {"task": {"type": "Retrieval"}, "dataset": {"name": "MTEB CovidRetrieval", "type": "C-MTEB/CovidRetrieval", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map_at_1", "value": 71.154}, {"type": "map_at_10", "value": 79.622}, {"type": "map_at_100", "value": 79.835}, {"type": "map_at_1000", "value": 79.838}, {"type": "map_at_3", "value": 77.678}, {"type": "map_at_5", "value": 78.999}, {"type": "mrr_at_1", "value": 71.444}, {"type": "mrr_at_10", "value": 79.679}, {"type": "mrr_at_100", "value": 79.892}, {"type": "mrr_at_1000", "value": 79.89500000000001}, {"type": "mrr_at_3", "value": 77.836}, {"type": "mrr_at_5", "value": 79.053}, {"type": "ndcg_at_1", "value": 71.444}, {"type": "ndcg_at_10", "value": 83.431}, {"type": "ndcg_at_100", "value": 84.379}, {"type": "ndcg_at_1000", "value": 84.45700000000001}, {"type": "ndcg_at_3", "value": 79.69000000000001}, {"type": "ndcg_at_5", "value": 81.989}, {"type": "precision_at_1", "value": 71.444}, {"type": "precision_at_10", "value": 9.621}, {"type": "precision_at_100", "value": 1.004}, {"type": "precision_at_1000", "value": 0.101}, {"type": "precision_at_3", "value": 28.662}, {"type": "precision_at_5", "value": 18.356}, {"type": "recall_at_1", "value": 71.154}, {"type": "recall_at_10", "value": 95.15299999999999}, {"type": "recall_at_100", "value": 99.368}, {"type": "recall_at_1000", "value": 100.0}, {"type": "recall_at_3", "value": 85.248}, {"type": "recall_at_5", "value": 90.806}]}, {"task": {"type": "Retrieval"}, "dataset": {"name": "MTEB DuRetrieval", "type": "C-MTEB/DuRetrieval", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map_at_1", "value": 25.267}, {"type": "map_at_10", "value": 75.467}, {"type": "map_at_100", "value": 78.57499999999999}, {"type": "map_at_1000", "value": 78.632}, {"type": "map_at_3", "value": 52.539}, {"type": "map_at_5", "value": 66.08699999999999}, {"type": "mrr_at_1", "value": 87.85}, {"type": "mrr_at_10", "value": 91.718}, {"type": "mrr_at_100", "value": 91.78399999999999}, {"type": "mrr_at_1000", "value": 91.788}, {"type": "mrr_at_3", "value": 91.333}, {"type": "mrr_at_5", "value": 91.556}, {"type": "ndcg_at_1", "value": 87.85}, {"type": "ndcg_at_10", "value": 83.92699999999999}, {"type": "ndcg_at_100", "value": 87.438}, {"type": "ndcg_at_1000", "value": 87.966}, {"type": "ndcg_at_3", "value": 82.951}, {"type": "ndcg_at_5", "value": 81.727}, {"type": "precision_at_1", "value": 87.85}, {"type": "precision_at_10", "value": 40.035}, {"type": "precision_at_100", "value": 4.741}, {"type": "precision_at_1000", "value": 0.48700000000000004}, {"type": "precision_at_3", "value": 74.033}, {"type": "precision_at_5", "value": 62.46000000000001}, {"type": "recall_at_1", "value": 25.267}, {"type": "recall_at_10", "value": 85.17099999999999}, {"type": "recall_at_100", "value": 96.032}, {"type": "recall_at_1000", "value": 98.788}, {"type": "recall_at_3", "value": 55.01200000000001}, {"type": "recall_at_5", "value": 71.195}]}, {"task": {"type": "Retrieval"}, "dataset": {"name": "MTEB EcomRetrieval", "type": "C-MTEB/EcomRetrieval", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map_at_1", "value": 51.1}, {"type": "map_at_10", "value": 61.107}, {"type": "map_at_100", "value": 61.61299999999999}, {"type": "map_at_1000", "value": 61.629999999999995}, {"type": "map_at_3", "value": 58.68299999999999}, {"type": "map_at_5", "value": 60.102999999999994}, {"type": "mrr_at_1", "value": 51.1}, {"type": "mrr_at_10", "value": 61.107}, {"type": "mrr_at_100", "value": 61.61299999999999}, {"type": "mrr_at_1000", "value": 61.629999999999995}, {"type": "mrr_at_3", "value": 58.68299999999999}, {"type": "mrr_at_5", "value": 60.102999999999994}, {"type": "ndcg_at_1", "value": 51.1}, {"type": "ndcg_at_10", "value": 66.11399999999999}, {"type": "ndcg_at_100", "value": 68.595}, {"type": "ndcg_at_1000", "value": 69.0}, {"type": "ndcg_at_3", "value": 61.129999999999995}, {"type": "ndcg_at_5", "value": 63.686}, {"type": "precision_at_1", "value": 51.1}, {"type": "precision_at_10", "value": 8.19}, {"type": "precision_at_100", "value": 0.936}, {"type": "precision_at_1000", "value": 0.097}, {"type": "precision_at_3", "value": 22.733}, {"type": "precision_at_5", "value": 14.879999999999999}, {"type": "recall_at_1", "value": 51.1}, {"type": "recall_at_10", "value": 81.89999999999999}, {"type": "recall_at_100", "value": 93.60000000000001}, {"type": "recall_at_1000", "value": 96.7}, {"type": "recall_at_3", "value": 68.2}, {"type": "recall_at_5", "value": 74.4}]}, {"task": {"type": "Classification"}, "dataset": {"name": "MTEB IFlyTek", "type": "C-MTEB/IFlyTek-classification", "config": "default", "split": "validation", "revision": "None"}, "metrics": [{"type": "accuracy", "value": 49.41131204309349}, {"type": "f1", "value": 37.687845728906524}]}, {"task": {"type": "Classification"}, "dataset": {"name": "MTEB JDReview", "type": "C-MTEB/JDReview-classification", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "accuracy", "value": 86.49155722326454}, {"type": "ap", "value": 55.102502318461134}, {"type": "f1", "value": 81.26055604335512}]}, {"task": {"type": "STS"}, "dataset": {"name": "MTEB LCQMC", "type": "C-MTEB/LCQMC", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "cos_sim_pearson", "value": 68.98090117218297}, {"type": "cos_sim_spearman", "value": 75.3115713301265}, {"type": "euclidean_pearson", "value": 74.17781110013063}, {"type": "euclidean_spearman", "value": 75.31156911148196}, {"type": "manhattan_pearson", "value": 74.16201018312145}, {"type": "manhattan_spearman", "value": 75.2877449602778}]}, {"task": {"type": "Reranking"}, "dataset": {"name": "MTEB MMarcoReranking", "type": "C-MTEB/Mmarco-reranking", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map", "value": 26.32558155922979}, {"type": "mrr", "value": 24.807936507936507}]}, {"task": {"type": "Retrieval"}, "dataset": {"name": "MTEB MMarcoRetrieval", "type": "C-MTEB/MMarcoRetrieval", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map_at_1", "value": 64.01299999999999}, {"type": "map_at_10", "value": 73.438}, {"type": "map_at_100", "value": 73.791}, {"type": "map_at_1000", "value": 73.804}, {"type": "map_at_3", "value": 71.54599999999999}, {"type": "map_at_5", "value": 72.75399999999999}, {"type": "mrr_at_1", "value": 66.189}, {"type": "mrr_at_10", "value": 74.098}, {"type": "mrr_at_100", "value": 74.404}, {"type": "mrr_at_1000", "value": 74.417}, {"type": "mrr_at_3", "value": 72.464}, {"type": "mrr_at_5", "value": 73.51100000000001}, {"type": "ndcg_at_1", "value": 66.189}, {"type": "ndcg_at_10", "value": 77.291}, {"type": "ndcg_at_100", "value": 78.884}, {"type": "ndcg_at_1000", "value": 79.239}, {"type": "ndcg_at_3", "value": 73.734}, {"type": "ndcg_at_5", "value": 75.759}, {"type": "precision_at_1", "value": 66.189}, {"type": "precision_at_10", "value": 9.401}, {"type": "precision_at_100", "value": 1.02}, {"type": "precision_at_1000", "value": 0.105}, {"type": "precision_at_3", "value": 27.851}, {"type": "precision_at_5", "value": 17.782}, {"type": "recall_at_1", "value": 64.01299999999999}, {"type": "recall_at_10", "value": 88.472}, {"type": "recall_at_100", "value": 95.707}, {"type": "recall_at_1000", "value": 98.48}, {"type": "recall_at_3", "value": 79.142}, {"type": "recall_at_5", "value": 83.91199999999999}]}, {"task": {"type": "Classification"}, "dataset": {"name": "MTEB MassiveIntentClassification (zh-CN)", "type": "mteb/amazon_massive_intent", "config": "zh-CN", "split": "test", "revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7"}, "metrics": [{"type": "accuracy", "value": 80.99529253530599}, {"type": "f1", "value": 78.35260600333876}]}, {"task": {"type": "Classification"}, "dataset": {"name": "MTEB MassiveScenarioClassification (zh-CN)", "type": "mteb/amazon_massive_scenario", "config": "zh-CN", "split": "test", "revision": "7d571f92784cd94a019292a1f45445077d0ef634"}, "metrics": [{"type": "accuracy", "value": 84.31069266980498}, {"type": "f1", "value": 84.20800214768072}]}, {"task": {"type": "Retrieval"}, "dataset": {"name": "MTEB MedicalRetrieval", "type": "C-MTEB/MedicalRetrieval", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map_at_1", "value": 51.300000000000004}, {"type": "map_at_10", "value": 58.119}, {"type": "map_at_100", "value": 58.662000000000006}, {"type": "map_at_1000", "value": 58.705}, {"type": "map_at_3", "value": 56.65}, {"type": "map_at_5", "value": 57.485}, {"type": "mrr_at_1", "value": 51.6}, {"type": "mrr_at_10", "value": 58.25900000000001}, {"type": "mrr_at_100", "value": 58.811}, {"type": "mrr_at_1000", "value": 58.85399999999999}, {"type": "mrr_at_3", "value": 56.8}, {"type": "mrr_at_5", "value": 57.635000000000005}, {"type": "ndcg_at_1", "value": 51.300000000000004}, {"type": "ndcg_at_10", "value": 61.400999999999996}, {"type": "ndcg_at_100", "value": 64.277}, {"type": "ndcg_at_1000", "value": 65.515}, {"type": "ndcg_at_3", "value": 58.361}, {"type": "ndcg_at_5", "value": 59.854}, {"type": "precision_at_1", "value": 51.300000000000004}, {"type": "precision_at_10", "value": 7.17}, {"type": "precision_at_100", "value": 0.857}, {"type": "precision_at_1000", "value": 0.096}, {"type": "precision_at_3", "value": 21.099999999999998}, {"type": "precision_at_5", "value": 13.38}, {"type": "recall_at_1", "value": 51.300000000000004}, {"type": "recall_at_10", "value": 71.7}, {"type": "recall_at_100", "value": 85.7}, {"type": "recall_at_1000", "value": 95.6}, {"type": "recall_at_3", "value": 63.3}, {"type": "recall_at_5", "value": 66.9}]}, {"task": {"type": "Classification"}, "dataset": {"name": "MTEB MultilingualSentiment", "type": "C-MTEB/MultilingualSentiment-classification", "config": "default", "split": "validation", "revision": "None"}, "metrics": [{"type": "accuracy", "value": 75.71666666666667}, {"type": "f1", "value": 75.66568156824309}]}, {"task": {"type": "PairClassification"}, "dataset": {"name": "MTEB Ocnli", "type": "C-MTEB/OCNLI", "config": "default", "split": "validation", "revision": "None"}, "metrics": [{"type": "cos_sim_accuracy", "value": 72.27937195452084}, {"type": "cos_sim_ap", "value": 75.45499247748975}, {"type": "cos_sim_f1", "value": 76.10008628127696}, {"type": "cos_sim_precision", "value": 64.33260393873084}, {"type": "cos_sim_recall", "value": 93.13621964097149}, {"type": "dot_accuracy", "value": 72.27937195452084}, {"type": "dot_ap", "value": 75.45505402270985}, {"type": "dot_f1", "value": 76.10008628127696}, {"type": "dot_precision", "value": 64.33260393873084}, {"type": "dot_recall", "value": 93.13621964097149}, {"type": "euclidean_accuracy", "value": 72.27937195452084}, {"type": "euclidean_ap", "value": 75.45499247748975}, {"type": "euclidean_f1", "value": 76.10008628127696}, {"type": "euclidean_precision", "value": 64.33260393873084}, {"type": "euclidean_recall", "value": 93.13621964097149}, {"type": "manhattan_accuracy", "value": 72.49593936112615}, {"type": "manhattan_ap", "value": 75.4461609306076}, {"type": "manhattan_f1", "value": 75.93856655290104}, {"type": "manhattan_precision", "value": 63.70794559770938}, {"type": "manhattan_recall", "value": 93.98099260823653}, {"type": "max_accuracy", "value": 72.49593936112615}, {"type": "max_ap", "value": 75.45505402270985}, {"type": "max_f1", "value": 76.10008628127696}]}, {"task": {"type": "Classification"}, "dataset": {"name": "MTEB OnlineShopping", "type": "C-MTEB/OnlineShopping-classification", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "accuracy", "value": 92.27000000000001}, {"type": "ap", "value": 90.49472745627727}, {"type": "f1", "value": 92.26384663599804}]}, {"task": {"type": "STS"}, "dataset": {"name": "MTEB PAWSX", "type": "C-MTEB/PAWSX", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "cos_sim_pearson", "value": 32.137716049584434}, {"type": "cos_sim_spearman", "value": 37.1857967995241}, {"type": "euclidean_pearson", "value": 37.2452443825464}, {"type": "euclidean_spearman", "value": 37.185796255670375}, {"type": "manhattan_pearson", "value": 37.24734607502982}, {"type": "manhattan_spearman", "value": 37.1488683590607}]}, {"task": {"type": "STS"}, "dataset": {"name": "MTEB QBQTC", "type": "C-MTEB/QBQTC", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "cos_sim_pearson", "value": 29.296851678412665}, {"type": "cos_sim_spearman", "value": 31.304998052183613}, {"type": "euclidean_pearson", "value": 29.732927229912416}, {"type": "euclidean_spearman", "value": 31.30492033806108}, {"type": "manhattan_pearson", "value": 29.93411926430157}, {"type": "manhattan_spearman", "value": 31.53088471232401}]}, {"task": {"type": "STS"}, "dataset": {"name": "MTEB STS22 (zh)", "type": "mteb/sts22-crosslingual-sts", "config": "zh", "split": "test", "revision": "6d1ba47164174a496b7fa5d3569dae26a6813b80"}, "metrics": [{"type": "cos_sim_pearson", "value": 57.36923700771426}, {"type": "cos_sim_spearman", "value": 65.60362646575587}, {"type": "euclidean_pearson", "value": 62.6478037667624}, {"type": "euclidean_spearman", "value": 65.60362716292573}, {"type": "manhattan_pearson", "value": 66.46506389106521}, {"type": "manhattan_spearman", "value": 66.03557886983012}]}, {"task": {"type": "STS"}, "dataset": {"name": "MTEB STSB", "type": "C-MTEB/STSB", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "cos_sim_pearson", "value": 78.59503403018131}, {"type": "cos_sim_spearman", "value": 78.63013154967143}, {"type": "euclidean_pearson", "value": 78.93410805371269}, {"type": "euclidean_spearman", "value": 78.63013154967143}, {"type": "manhattan_pearson", "value": 78.89417119805393}, {"type": "manhattan_spearman", "value": 78.59414974181517}]}, {"task": {"type": "Reranking"}, "dataset": {"name": "MTEB T2Reranking", "type": "C-MTEB/T2Reranking", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map", "value": 66.28738004133163}, {"type": "mrr", "value": 76.13161062213194}]}, {"task": {"type": "Retrieval"}, "dataset": {"name": "MTEB T2Retrieval", "type": "C-MTEB/T2Retrieval", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map_at_1", "value": 26.588}, {"type": "map_at_10", "value": 71.705}, {"type": "map_at_100", "value": 75.787}, {"type": "map_at_1000", "value": 75.875}, {"type": "map_at_3", "value": 50.942}, {"type": "map_at_5", "value": 62.038000000000004}, {"type": "mrr_at_1", "value": 86.823}, {"type": "mrr_at_10", "value": 90.72}, {"type": "mrr_at_100", "value": 90.814}, {"type": "mrr_at_1000", "value": 90.81700000000001}, {"type": "mrr_at_3", "value": 90.153}, {"type": "mrr_at_5", "value": 90.53099999999999}, {"type": "ndcg_at_1", "value": 86.823}, {"type": "ndcg_at_10", "value": 80.83200000000001}, {"type": "ndcg_at_100", "value": 85.5}, {"type": "ndcg_at_1000", "value": 86.287}, {"type": "ndcg_at_3", "value": 82.178}, {"type": "ndcg_at_5", "value": 80.557}, {"type": "precision_at_1", "value": 86.823}, {"type": "precision_at_10", "value": 39.708}, {"type": "precision_at_100", "value": 4.951}, {"type": "precision_at_1000", "value": 0.515}, {"type": "precision_at_3", "value": 71.283}, {"type": "precision_at_5", "value": 59.28600000000001}, {"type": "recall_at_1", "value": 26.588}, {"type": "recall_at_10", "value": 80.488}, {"type": "recall_at_100", "value": 94.774}, {"type": "recall_at_1000", "value": 98.545}, {"type": "recall_at_3", "value": 53.43}, {"type": "recall_at_5", "value": 66.80199999999999}]}, {"task": {"type": "Classification"}, "dataset": {"name": "MTEB TNews", "type": "C-MTEB/TNews-classification", "config": "default", "split": "validation", "revision": "None"}, "metrics": [{"type": "accuracy", "value": 52.580000000000005}, {"type": "f1", "value": 50.85086643723326}]}, {"task": {"type": "Clustering"}, "dataset": {"name": "MTEB ThuNewsClusteringP2P", "type": "C-MTEB/ThuNewsClusteringP2P", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "v_measure", "value": 72.0633362729503}]}, {"task": {"type": "Clustering"}, "dataset": {"name": "MTEB ThuNewsClusteringS2S", "type": "C-MTEB/ThuNewsClusteringS2S", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "v_measure", "value": 66.85958656370434}]}, {"task": {"type": "Retrieval"}, "dataset": {"name": "MTEB VideoRetrieval", "type": "C-MTEB/VideoRetrieval", "config": "default", "split": "dev", "revision": "None"}, "metrics": [{"type": "map_at_1", "value": 57.199999999999996}, {"type": "map_at_10", "value": 66.628}, {"type": "map_at_100", "value": 67.107}, {"type": "map_at_1000", "value": 67.121}, {"type": "map_at_3", "value": 64.633}, {"type": "map_at_5", "value": 65.813}, {"type": "mrr_at_1", "value": 57.199999999999996}, {"type": "mrr_at_10", "value": 66.628}, {"type": "mrr_at_100", "value": 67.107}, {"type": "mrr_at_1000", "value": 67.121}, {"type": "mrr_at_3", "value": 64.633}, {"type": "mrr_at_5", "value": 65.813}, {"type": "ndcg_at_1", "value": 57.199999999999996}, {"type": "ndcg_at_10", "value": 71.029}, {"type": "ndcg_at_100", "value": 73.289}, {"type": "ndcg_at_1000", "value": 73.68599999999999}, {"type": "ndcg_at_3", "value": 66.945}, {"type": "ndcg_at_5", "value": 69.037}, {"type": "precision_at_1", "value": 57.199999999999996}, {"type": "precision_at_10", "value": 8.48}, {"type": "precision_at_100", "value": 0.9520000000000001}, {"type": "precision_at_1000", "value": 0.098}, {"type": "precision_at_3", "value": 24.532999999999998}, {"type": "precision_at_5", "value": 15.72}, {"type": "recall_at_1", "value": 57.199999999999996}, {"type": "recall_at_10", "value": 84.8}, {"type": "recall_at_100", "value": 95.19999999999999}, {"type": "recall_at_1000", "value": 98.4}, {"type": "recall_at_3", "value": 73.6}, {"type": "recall_at_5", "value": 78.60000000000001}]}, {"task": {"type": "Classification"}, "dataset": {"name": "MTEB Waimai", "type": "C-MTEB/waimai-classification", "config": "default", "split": "test", "revision": "None"}, "metrics": [{"type": "accuracy", "value": 88.44000000000001}, {"type": "ap", "value": 73.4626403421891}, {"type": "f1", "value": 86.74820378279573}]}]}]}
sentence-transformers pytorch bert mteb feature-extraction sentence-similarity model-index endpoints_compatible region:us