diff --git "a/README.md" "b/README.md"
--- "a/README.md"
+++ "b/README.md"
@@ -1,26 +1,24 @@
---
tags:
- mteb
-- sentence-transformers
-- transformers
model-index:
-- name: SFR-Embedding-Mistral
+- name: Salesforce/SFR-Embedding-2_R
results:
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
- name: MTEB AmazonCounterfactualClassification (en)
- config: en
+ name: MTEB AmazonCounterfactualClassification
+ config: default
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
- value: 77.92537313432834
+ value: 92.71641791044776
- type: ap
- value: 40.86767661556651
+ value: 69.47931007147756
- type: f1
- value: 71.65758897929837
+ value: 88.0252625393374
- task:
type: Classification
dataset:
@@ -31,193 +29,243 @@ model-index:
revision: e2d317d38cd51312af73b3d32a06d1a08b442046
metrics:
- type: accuracy
- value: 95.967
+ value: 97.31075
- type: ap
- value: 94.46300829592593
+ value: 96.26693923450127
- type: f1
- value: 95.96507173189292
+ value: 97.31042448894502
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
- name: MTEB AmazonReviewsClassification (en)
- config: en
+ name: MTEB AmazonReviewsClassification
+ config: default
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
- value: 54.352000000000004
+ value: 61.040000000000006
- type: f1
- value: 53.636682615380174
+ value: 60.78646832640785
- task:
type: Retrieval
dataset:
- type: arguana
+ type: mteb/arguana
name: MTEB ArguAna
config: default
split: test
- revision: None
+ revision: c22ab2a51041ffd869aaddef7af8d8215647e41a
metrics:
- - type: ndcg_at_1
- value: 43.314
- - type: ndcg_at_2
- value: 54.757
- - type: ndcg_at_3
- value: 58.84700000000001
- - type: ndcg_at_5
- value: 63.634
- - type: ndcg_at_7
- value: 65.741
- - type: ndcg_at_10
- value: 67.171
- - type: ndcg_at_20
- value: 68.585
- - type: ndcg_at_30
- value: 68.81
- - type: ndcg_at_50
- value: 68.932
- - type: ndcg_at_70
- value: 68.992
- - type: ndcg_at_100
- value: 69.014
- - type: ndcg_at_200
- value: 69.014
- - type: ndcg_at_300
- value: 69.014
- - type: ndcg_at_500
- value: 69.014
- - type: ndcg_at_700
- value: 69.014
- - type: ndcg_at_1000
- value: 69.014
- type: map_at_1
- value: 43.314
- - type: map_at_2
- value: 52.383
- - type: map_at_3
- value: 55.108999999999995
- - type: map_at_5
- value: 57.772999999999996
- - type: map_at_7
- value: 58.718
+ value: 37.767
- type: map_at_10
- value: 59.256
- - type: map_at_20
- value: 59.668
- - type: map_at_30
- value: 59.709999999999994
- - type: map_at_50
- value: 59.727
- - type: map_at_70
- value: 59.733999999999995
+ value: 53.908
- type: map_at_100
- value: 59.73500000000001
- - type: map_at_200
- value: 59.73500000000001
- - type: map_at_300
- value: 59.73500000000001
- - type: map_at_500
- value: 59.73500000000001
- - type: map_at_700
- value: 59.73500000000001
+ value: 54.583000000000006
- type: map_at_1000
- value: 59.73500000000001
- - type: recall_at_1
- value: 43.314
- - type: recall_at_2
- value: 61.451
- - type: recall_at_3
- value: 69.63000000000001
- - type: recall_at_5
- value: 81.223
- - type: recall_at_7
- value: 87.33999999999999
- - type: recall_at_10
- value: 92.034
- - type: recall_at_20
- value: 97.44
- - type: recall_at_30
- value: 98.506
- - type: recall_at_50
- value: 99.14699999999999
- - type: recall_at_70
- value: 99.502
- - type: recall_at_100
- value: 99.644
- - type: recall_at_200
- value: 99.644
- - type: recall_at_300
- value: 99.644
- - type: recall_at_500
- value: 99.644
- - type: recall_at_700
- value: 99.644
- - type: recall_at_1000
- value: 99.644
+ value: 54.583999999999996
+ - type: map_at_20
+ value: 54.50899999999999
+ - type: map_at_3
+ value: 49.514
+ - type: map_at_5
+ value: 52.059999999999995
+ - type: mrr_at_1
+ value: 38.26458036984353
+ - type: mrr_at_10
+ value: 54.120408001987066
+ - type: mrr_at_100
+ value: 54.780719904297406
+ - type: mrr_at_1000
+ value: 54.78174226698592
+ - type: mrr_at_20
+ value: 54.706604527160295
+ - type: mrr_at_3
+ value: 49.71550497866294
+ - type: mrr_at_5
+ value: 52.247510668563436
+ - type: nauc_map_at_1000_diff1
+ value: 11.892288101259679
+ - type: nauc_map_at_1000_max
+ value: -12.510113256704939
+ - type: nauc_map_at_100_diff1
+ value: 11.892982473311001
+ - type: nauc_map_at_100_max
+ value: -12.508290193061327
+ - type: nauc_map_at_10_diff1
+ value: 11.651977254362553
+ - type: nauc_map_at_10_max
+ value: -12.330934006894392
+ - type: nauc_map_at_1_diff1
+ value: 15.715603626122421
+ - type: nauc_map_at_1_max
+ value: -15.078886483885878
+ - type: nauc_map_at_20_diff1
+ value: 11.862780666227493
+ - type: nauc_map_at_20_max
+ value: -12.426347384400083
+ - type: nauc_map_at_3_diff1
+ value: 11.890344562501575
+ - type: nauc_map_at_3_max
+ value: -12.500252247101235
+ - type: nauc_map_at_5_diff1
+ value: 11.312005111039689
+ - type: nauc_map_at_5_max
+ value: -12.079778779333896
+ - type: nauc_mrr_at_1000_diff1
+ value: 10.428236648105853
+ - type: nauc_mrr_at_1000_max
+ value: -13.017554567405737
+ - type: nauc_mrr_at_100_diff1
+ value: 10.428980359732561
+ - type: nauc_mrr_at_100_max
+ value: -13.015717756577308
+ - type: nauc_mrr_at_10_diff1
+ value: 10.220148638327622
+ - type: nauc_mrr_at_10_max
+ value: -12.848991430415161
+ - type: nauc_mrr_at_1_diff1
+ value: 14.294348082431762
+ - type: nauc_mrr_at_1_max
+ value: -15.040983031885933
+ - type: nauc_mrr_at_20_diff1
+ value: 10.402818393818745
+ - type: nauc_mrr_at_20_max
+ value: -12.93241236123898
+ - type: nauc_mrr_at_3_diff1
+ value: 10.477325916240387
+ - type: nauc_mrr_at_3_max
+ value: -13.110117779394702
+ - type: nauc_mrr_at_5_diff1
+ value: 9.834612447153244
+ - type: nauc_mrr_at_5_max
+ value: -12.6504294436756
+ - type: nauc_ndcg_at_1000_diff1
+ value: 11.353977595620494
+ - type: nauc_ndcg_at_1000_max
+ value: -12.037808007008312
+ - type: nauc_ndcg_at_100_diff1
+ value: 11.360271960765596
+ - type: nauc_ndcg_at_100_max
+ value: -12.005462095904923
+ - type: nauc_ndcg_at_10_diff1
+ value: 10.283725379801774
+ - type: nauc_ndcg_at_10_max
+ value: -11.12234797539797
+ - type: nauc_ndcg_at_1_diff1
+ value: 15.715603626122421
+ - type: nauc_ndcg_at_1_max
+ value: -15.078886483885878
+ - type: nauc_ndcg_at_20_diff1
+ value: 11.16839538484597
+ - type: nauc_ndcg_at_20_max
+ value: -11.296110432329098
+ - type: nauc_ndcg_at_3_diff1
+ value: 10.728469173159565
+ - type: nauc_ndcg_at_3_max
+ value: -11.6907053741208
+ - type: nauc_ndcg_at_5_diff1
+ value: 9.4939298650561
+ - type: nauc_ndcg_at_5_max
+ value: -10.761365220540615
+ - type: nauc_precision_at_1000_diff1
+ value: -13.166980717158044
+ - type: nauc_precision_at_1000_max
+ value: 11.937573391343543
+ - type: nauc_precision_at_100_diff1
+ value: -5.036875569003799
+ - type: nauc_precision_at_100_max
+ value: 14.762952912852775
+ - type: nauc_precision_at_10_diff1
+ value: 0.07806243776481814
+ - type: nauc_precision_at_10_max
+ value: -1.0662999457924391
+ - type: nauc_precision_at_1_diff1
+ value: 15.715603626122421
+ - type: nauc_precision_at_1_max
+ value: -15.078886483885878
+ - type: nauc_precision_at_20_diff1
+ value: 1.9311706633140098
+ - type: nauc_precision_at_20_max
+ value: 21.626041514551105
+ - type: nauc_precision_at_3_diff1
+ value: 6.965461597850012
+ - type: nauc_precision_at_3_max
+ value: -9.058131095347413
+ - type: nauc_precision_at_5_diff1
+ value: 1.751884489911988
+ - type: nauc_precision_at_5_max
+ value: -5.090281000735729
+ - type: nauc_recall_at_1000_diff1
+ value: -13.166980717160087
+ - type: nauc_recall_at_1000_max
+ value: 11.937573391339985
+ - type: nauc_recall_at_100_diff1
+ value: -5.0368755690026905
+ - type: nauc_recall_at_100_max
+ value: 14.76295291285537
+ - type: nauc_recall_at_10_diff1
+ value: 0.07806243776481815
+ - type: nauc_recall_at_10_max
+ value: -1.066299945792319
+ - type: nauc_recall_at_1_diff1
+ value: 15.715603626122421
+ - type: nauc_recall_at_1_max
+ value: -15.078886483885878
+ - type: nauc_recall_at_20_diff1
+ value: 1.931170663315172
+ - type: nauc_recall_at_20_max
+ value: 21.6260415145515
+ - type: nauc_recall_at_3_diff1
+ value: 6.965461597850088
+ - type: nauc_recall_at_3_max
+ value: -9.058131095347365
+ - type: nauc_recall_at_5_diff1
+ value: 1.7518844899120165
+ - type: nauc_recall_at_5_max
+ value: -5.090281000735665
+ - type: ndcg_at_1
+ value: 37.767
+ - type: ndcg_at_10
+ value: 62.339999999999996
+ - type: ndcg_at_100
+ value: 64.89399999999999
+ - type: ndcg_at_1000
+ value: 64.914
+ - type: ndcg_at_20
+ value: 64.402
+ - type: ndcg_at_3
+ value: 53.33
+ - type: ndcg_at_5
+ value: 57.93899999999999
- type: precision_at_1
- value: 43.314
- - type: precision_at_2
- value: 30.725
- - type: precision_at_3
- value: 23.21
- - type: precision_at_5
- value: 16.245
- - type: precision_at_7
- value: 12.477
+ value: 37.767
- type: precision_at_10
- value: 9.203
- - type: precision_at_20
- value: 4.872
- - type: precision_at_30
- value: 3.2840000000000003
- - type: precision_at_50
- value: 1.983
- - type: precision_at_70
- value: 1.421
+ value: 8.905000000000001
- type: precision_at_100
- value: 0.996
- - type: precision_at_200
- value: 0.498
- - type: precision_at_300
- value: 0.332
- - type: precision_at_500
- value: 0.199
- - type: precision_at_700
- value: 0.14200000000000002
+ value: 0.9950000000000001
- type: precision_at_1000
value: 0.1
- - type: mrr_at_1
- value: 44.666
- - type: mrr_at_2
- value: 52.418
- - type: mrr_at_3
- value: 55.595000000000006
- - type: mrr_at_5
- value: 58.205
- - type: mrr_at_7
- value: 59.202999999999996
- - type: mrr_at_10
- value: 59.727
- - type: mrr_at_20
- value: 60.133
- - type: mrr_at_30
- value: 60.178
- - type: mrr_at_50
- value: 60.192
- - type: mrr_at_70
- value: 60.19799999999999
- - type: mrr_at_100
- value: 60.199999999999996
- - type: mrr_at_200
- value: 60.199999999999996
- - type: mrr_at_300
- value: 60.199999999999996
- - type: mrr_at_500
- value: 60.199999999999996
- - type: mrr_at_700
- value: 60.199999999999996
- - type: mrr_at_1000
- value: 60.199999999999996
+ - type: precision_at_20
+ value: 4.8469999999999995
+ - type: precision_at_3
+ value: 21.456
+ - type: precision_at_5
+ value: 15.121
+ - type: recall_at_1
+ value: 37.767
+ - type: recall_at_10
+ value: 89.047
+ - type: recall_at_100
+ value: 99.502
+ - type: recall_at_1000
+ value: 99.644
+ - type: recall_at_20
+ value: 96.942
+ - type: recall_at_3
+ value: 64.36699999999999
+ - type: recall_at_5
+ value: 75.605
- task:
type: Clustering
dataset:
@@ -228,7 +276,7 @@ model-index:
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d
metrics:
- type: v_measure
- value: 52.07508593014336
+ value: 54.024325012036314
- task:
type: Clustering
dataset:
@@ -239,7 +287,7 @@ model-index:
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53
metrics:
- type: v_measure
- value: 47.381339333240675
+ value: 48.817300846601675
- task:
type: Reranking
dataset:
@@ -250,9 +298,17 @@ model-index:
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
metrics:
- type: map
- value: 67.58376647859171
+ value: 66.71478959728732
- type: mrr
- value: 80.56885635140483
+ value: 79.07202216066482
+ - type: nAUC_map_diff1
+ value: 12.107502181210737
+ - type: nAUC_map_max
+ value: 26.88330114153595
+ - type: nAUC_mrr_diff1
+ value: 21.07548754737264
+ - type: nAUC_mrr_max
+ value: 43.7325419146008
- task:
type: STS
dataset:
@@ -263,17 +319,17 @@ model-index:
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
metrics:
- type: cos_sim_pearson
- value: 88.40107280274783
+ value: 88.79517914982239
- type: cos_sim_spearman
- value: 86.07003345325681
+ value: 87.60440576436838
- type: euclidean_pearson
- value: 87.1726034325395
+ value: 87.75596873521118
- type: euclidean_spearman
- value: 86.07003345325681
+ value: 87.60440576436838
- type: manhattan_pearson
- value: 87.25660625029772
+ value: 87.74113773865973
- type: manhattan_spearman
- value: 86.3808839096893
+ value: 87.50560833247899
- task:
type: Classification
dataset:
@@ -284,9 +340,9 @@ model-index:
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
metrics:
- type: accuracy
- value: 88.81168831168831
+ value: 90.02272727272727
- type: f1
- value: 88.76514496560141
+ value: 89.96681880265936
- task:
type: Clustering
dataset:
@@ -297,7 +353,7 @@ model-index:
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40
metrics:
- type: v_measure
- value: 43.9382520874344
+ value: 50.75930389699286
- task:
type: Clustering
dataset:
@@ -308,514 +364,664 @@ model-index:
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908
metrics:
- type: v_measure
- value: 41.14351847240913
+ value: 46.57286439805565
- task:
type: Retrieval
dataset:
- type: BeIR/cqadupstack
- name: MTEB CQADupstackRetrieval
+ type: mteb/cqadupstack-android
+ name: MTEB CQADupstackAndroidRetrieval
config: default
split: test
- revision: None
+ revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a
metrics:
- - type: ndcg_at_1
- value: 34.51166666666667
- - type: ndcg_at_2
- value: 38.51591666666667
- - type: ndcg_at_3
- value: 40.95083333333333
- - type: ndcg_at_5
- value: 43.580666666666666
- - type: ndcg_at_7
- value: 45.0625
- - type: ndcg_at_10
- value: 46.49083333333333
- - type: ndcg_at_20
- value: 48.731333333333325
- - type: ndcg_at_30
- value: 49.78666666666667
- - type: ndcg_at_50
- value: 50.84049999999999
- - type: ndcg_at_70
- value: 51.393750000000004
- - type: ndcg_at_100
- value: 51.883333333333326
- - type: ndcg_at_200
- value: 52.65225
- - type: ndcg_at_300
- value: 52.98241666666669
- - type: ndcg_at_500
- value: 53.28541666666668
- - type: ndcg_at_700
- value: 53.49241666666668
- - type: ndcg_at_1000
- value: 53.63758333333334
- type: map_at_1
- value: 29.10075
- - type: map_at_2
- value: 34.636500000000005
- - type: map_at_3
- value: 36.92033333333333
- - type: map_at_5
- value: 38.81641666666666
- - type: map_at_7
- value: 39.635416666666664
+ value: 28.056666666666665
- type: map_at_10
- value: 40.294583333333335
- - type: map_at_20
- value: 41.07574999999999
- - type: map_at_30
- value: 41.333
- - type: map_at_50
- value: 41.529333333333334
- - type: map_at_70
- value: 41.606833333333334
+ value: 39.61749999999999
- type: map_at_100
- value: 41.66224999999999
- - type: map_at_200
- value: 41.72691666666666
- - type: map_at_300
- value: 41.746583333333334
- - type: map_at_500
- value: 41.75983333333333
- - type: map_at_700
- value: 41.76558333333333
+ value: 41.00666666666666
- type: map_at_1000
- value: 41.769000000000005
- - type: recall_at_1
- value: 29.10075
- - type: recall_at_2
- value: 39.07658333333333
- - type: recall_at_3
- value: 44.93591666666667
- - type: recall_at_5
- value: 51.66883333333333
- - type: recall_at_7
- value: 55.881000000000014
- - type: recall_at_10
- value: 60.34691666666667
- - type: recall_at_20
- value: 68.44016666666667
- - type: recall_at_30
- value: 72.90766666666667
- - type: recall_at_50
- value: 77.843
- - type: recall_at_70
- value: 80.70366666666668
- - type: recall_at_100
- value: 83.42866666666667
- - type: recall_at_200
- value: 88.06816666666668
- - type: recall_at_300
- value: 90.249
- - type: recall_at_500
- value: 92.37616666666668
- - type: recall_at_700
- value: 93.978
- - type: recall_at_1000
- value: 95.12791666666666
- - type: precision_at_1
- value: 34.51166666666667
- - type: precision_at_2
- value: 24.326333333333327
- - type: precision_at_3
- value: 19.099249999999998
- - type: precision_at_5
- value: 13.672666666666666
- - type: precision_at_7
- value: 10.772
- - type: precision_at_10
- value: 8.302166666666668
- - type: precision_at_20
- value: 4.8960833333333325
- - type: precision_at_30
- value: 3.551083333333333
- - type: precision_at_50
- value: 2.3386666666666662
- - type: precision_at_70
- value: 1.7605833333333334
- - type: precision_at_100
- value: 1.2965
- - type: precision_at_200
- value: 0.7106666666666668
- - type: precision_at_300
- value: 0.4955
- - type: precision_at_500
- value: 0.3106666666666667
- - type: precision_at_700
- value: 0.22791666666666668
- - type: precision_at_1000
- value: 0.1635833333333333
+ value: 41.11358333333334
+ - type: map_at_20
+ value: 40.410250000000005
+ - type: map_at_3
+ value: 35.98591666666667
+ - type: map_at_5
+ value: 38.02
- type: mrr_at_1
- value: 34.51166666666667
- - type: mrr_at_2
- value: 39.954249999999995
- - type: mrr_at_3
- value: 41.93741666666668
- - type: mrr_at_5
- value: 43.487166666666674
- - type: mrr_at_7
- value: 44.14983333333333
+ value: 33.73950708467142
- type: mrr_at_10
- value: 44.62766666666666
- - type: mrr_at_20
- value: 45.15291666666668
- - type: mrr_at_30
- value: 45.317
- - type: mrr_at_50
- value: 45.42875
- - type: mrr_at_70
- value: 45.46966666666667
+ value: 44.0987162763402
- type: mrr_at_100
- value: 45.49716666666667
- - type: mrr_at_200
- value: 45.525166666666664
- - type: mrr_at_300
- value: 45.53233333333335
- - type: mrr_at_500
- value: 45.5365
- - type: mrr_at_700
- value: 45.538583333333335
+ value: 44.94302678553521
- type: mrr_at_1000
- value: 45.539583333333326
+ value: 44.98758207055161
+ - type: mrr_at_20
+ value: 44.61156907536121
+ - type: mrr_at_3
+ value: 41.247253732468415
+ - type: mrr_at_5
+ value: 42.84859071071954
+ - type: nauc_map_at_1000_diff1
+ value: 39.6443709527807
+ - type: nauc_map_at_1000_max
+ value: 28.10414203846302
+ - type: nauc_map_at_100_diff1
+ value: 39.64380393432371
+ - type: nauc_map_at_100_max
+ value: 28.092378920173438
+ - type: nauc_map_at_10_diff1
+ value: 39.66391719012231
+ - type: nauc_map_at_10_max
+ value: 27.56524015820569
+ - type: nauc_map_at_1_diff1
+ value: 45.15339300635892
+ - type: nauc_map_at_1_max
+ value: 24.89426197328438
+ - type: nauc_map_at_20_diff1
+ value: 39.593013993947444
+ - type: nauc_map_at_20_max
+ value: 27.872168151457345
+ - type: nauc_map_at_3_diff1
+ value: 40.35439641775099
+ - type: nauc_map_at_3_max
+ value: 26.462314926374873
+ - type: nauc_map_at_5_diff1
+ value: 39.844820169914136
+ - type: nauc_map_at_5_max
+ value: 27.11003823044208
+ - type: nauc_mrr_at_1000_diff1
+ value: 39.2511516032393
+ - type: nauc_mrr_at_1000_max
+ value: 30.36370456611554
+ - type: nauc_mrr_at_100_diff1
+ value: 39.23557979126079
+ - type: nauc_mrr_at_100_max
+ value: 30.36108511075049
+ - type: nauc_mrr_at_10_diff1
+ value: 39.20124905182319
+ - type: nauc_mrr_at_10_max
+ value: 30.29557925026432
+ - type: nauc_mrr_at_1_diff1
+ value: 43.94309065090076
+ - type: nauc_mrr_at_1_max
+ value: 30.11794947329922
+ - type: nauc_mrr_at_20_diff1
+ value: 39.18748361887478
+ - type: nauc_mrr_at_20_max
+ value: 30.338708586578413
+ - type: nauc_mrr_at_3_diff1
+ value: 39.58562944544491
+ - type: nauc_mrr_at_3_max
+ value: 30.05222360714671
+ - type: nauc_mrr_at_5_diff1
+ value: 39.29184662434337
+ - type: nauc_mrr_at_5_max
+ value: 30.31134275619768
+ - type: nauc_ndcg_at_1000_diff1
+ value: 38.210143189277275
+ - type: nauc_ndcg_at_1000_max
+ value: 29.808772620631363
+ - type: nauc_ndcg_at_100_diff1
+ value: 37.886399370479225
+ - type: nauc_ndcg_at_100_max
+ value: 29.789191789703757
+ - type: nauc_ndcg_at_10_diff1
+ value: 37.7150996312033
+ - type: nauc_ndcg_at_10_max
+ value: 28.634133121074456
+ - type: nauc_ndcg_at_1_diff1
+ value: 43.94309065090076
+ - type: nauc_ndcg_at_1_max
+ value: 30.11794947329922
+ - type: nauc_ndcg_at_20_diff1
+ value: 37.49835643052173
+ - type: nauc_ndcg_at_20_max
+ value: 29.158925896563886
+ - type: nauc_ndcg_at_3_diff1
+ value: 38.65825315631316
+ - type: nauc_ndcg_at_3_max
+ value: 28.146443378716608
+ - type: nauc_ndcg_at_5_diff1
+ value: 38.06434768442492
+ - type: nauc_ndcg_at_5_max
+ value: 28.3653998051417
+ - type: nauc_precision_at_1000_diff1
+ value: -7.645368303985066
+ - type: nauc_precision_at_1000_max
+ value: 6.616811451768115
+ - type: nauc_precision_at_100_diff1
+ value: -1.6477228809639735
+ - type: nauc_precision_at_100_max
+ value: 16.185093755306525
+ - type: nauc_precision_at_10_diff1
+ value: 12.349305719091785
+ - type: nauc_precision_at_10_max
+ value: 25.60845652792053
+ - type: nauc_precision_at_1_diff1
+ value: 43.94309065090076
+ - type: nauc_precision_at_1_max
+ value: 30.11794947329922
+ - type: nauc_precision_at_20_diff1
+ value: 6.455114392744436
+ - type: nauc_precision_at_20_max
+ value: 23.320434279068596
+ - type: nauc_precision_at_3_diff1
+ value: 25.425158020184274
+ - type: nauc_precision_at_3_max
+ value: 29.080117669843062
+ - type: nauc_precision_at_5_diff1
+ value: 19.000618451734592
+ - type: nauc_precision_at_5_max
+ value: 27.83747968465858
+ - type: nauc_recall_at_1000_diff1
+ value: 21.501453539687805
+ - type: nauc_recall_at_1000_max
+ value: 49.33460231674802
+ - type: nauc_recall_at_100_diff1
+ value: 26.248723616973567
+ - type: nauc_recall_at_100_max
+ value: 33.66516179273834
+ - type: nauc_recall_at_10_diff1
+ value: 29.942088542998626
+ - type: nauc_recall_at_10_max
+ value: 26.184929767257476
+ - type: nauc_recall_at_1_diff1
+ value: 45.15339300635892
+ - type: nauc_recall_at_1_max
+ value: 24.89426197328438
+ - type: nauc_recall_at_20_diff1
+ value: 27.874466917875047
+ - type: nauc_recall_at_20_max
+ value: 28.061626901296975
+ - type: nauc_recall_at_3_diff1
+ value: 34.40693497233646
+ - type: nauc_recall_at_3_max
+ value: 24.853632891165372
+ - type: nauc_recall_at_5_diff1
+ value: 32.09207205145065
+ - type: nauc_recall_at_5_max
+ value: 25.755493506809007
+ - type: ndcg_at_1
+ value: 33.739666666666665
+ - type: ndcg_at_10
+ value: 46.10683333333334
+ - type: ndcg_at_100
+ value: 51.49275000000001
+ - type: ndcg_at_1000
+ value: 53.2585
+ - type: ndcg_at_20
+ value: 48.349
+ - type: ndcg_at_3
+ value: 40.12416666666667
+ - type: ndcg_at_5
+ value: 42.94783333333333
+ - type: precision_at_1
+ value: 33.739666666666665
+ - type: precision_at_10
+ value: 8.46025
+ - type: precision_at_100
+ value: 1.3215833333333333
+ - type: precision_at_1000
+ value: 0.16524999999999998
+ - type: precision_at_20
+ value: 4.9935833333333335
+ - type: precision_at_3
+ value: 19.00516666666667
+ - type: precision_at_5
+ value: 13.72141666666667
+ - type: recall_at_1
+ value: 28.056666666666665
+ - type: recall_at_10
+ value: 60.68825000000001
+ - type: recall_at_100
+ value: 83.74433333333334
+ - type: recall_at_1000
+ value: 95.62299999999999
+ - type: recall_at_20
+ value: 68.77641666666668
+ - type: recall_at_3
+ value: 44.06991666666667
+ - type: recall_at_5
+ value: 51.324999999999996
- task:
type: Retrieval
dataset:
- type: climate-fever
+ type: mteb/climate-fever
name: MTEB ClimateFEVER
config: default
split: test
- revision: None
+ revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380
metrics:
- - type: ndcg_at_1
- value: 35.179
- - type: ndcg_at_2
- value: 31.243
- - type: ndcg_at_3
- value: 30.562
- - type: ndcg_at_5
- value: 32.409
- - type: ndcg_at_7
- value: 34.525
- - type: ndcg_at_10
- value: 36.415
- - type: ndcg_at_20
- value: 39.443
- - type: ndcg_at_30
- value: 40.796
- - type: ndcg_at_50
- value: 42.16
- - type: ndcg_at_70
- value: 42.971
- - type: ndcg_at_100
- value: 43.691
- - type: ndcg_at_200
- value: 45.004
- - type: ndcg_at_300
- value: 45.527
- - type: ndcg_at_500
- value: 46.072
- - type: ndcg_at_700
- value: 46.387
- - type: ndcg_at_1000
- value: 46.663
- type: map_at_1
- value: 15.692
- - type: map_at_2
- value: 20.116
- - type: map_at_3
- value: 22.6
- - type: map_at_5
- value: 24.701
- - type: map_at_7
- value: 25.934
+ value: 15.609
- type: map_at_10
- value: 26.843
- - type: map_at_20
- value: 27.975
- - type: map_at_30
- value: 28.372000000000003
- - type: map_at_50
- value: 28.671000000000003
- - type: map_at_70
- value: 28.803
+ value: 25.584
- type: map_at_100
- value: 28.895
- - type: map_at_200
- value: 29.011
- - type: map_at_300
- value: 29.042
- - type: map_at_500
- value: 29.065
- - type: map_at_700
- value: 29.075
+ value: 27.291999999999998
- type: map_at_1000
- value: 29.081000000000003
- - type: recall_at_1
- value: 15.692
- - type: recall_at_2
- value: 22.602
- - type: recall_at_3
- value: 27.814
- - type: recall_at_5
- value: 33.756
- - type: recall_at_7
- value: 38.073
- - type: recall_at_10
- value: 42.553000000000004
- - type: recall_at_20
- value: 51.121
- - type: recall_at_30
- value: 55.523999999999994
- - type: recall_at_50
- value: 60.586
- - type: recall_at_70
- value: 63.94
- - type: recall_at_100
- value: 67.134
- - type: recall_at_200
- value: 73.543
- - type: recall_at_300
- value: 76.372
- - type: recall_at_500
- value: 79.60199999999999
- - type: recall_at_700
- value: 81.536
- - type: recall_at_1000
- value: 83.37400000000001
- - type: precision_at_1
- value: 35.179
- - type: precision_at_2
- value: 27.199
- - type: precision_at_3
- value: 22.953000000000003
- - type: precision_at_5
- value: 17.224999999999998
- - type: precision_at_7
- value: 14.238999999999999
- - type: precision_at_10
- value: 11.303
- - type: precision_at_20
- value: 6.954000000000001
- - type: precision_at_30
- value: 5.116
- - type: precision_at_50
- value: 3.395
- - type: precision_at_70
- value: 2.579
- - type: precision_at_100
- value: 1.9109999999999998
- - type: precision_at_200
- value: 1.065
- - type: precision_at_300
- value: 0.743
- - type: precision_at_500
- value: 0.46699999999999997
- - type: precision_at_700
- value: 0.344
- - type: precision_at_1000
- value: 0.247
+ value: 27.471
+ - type: map_at_20
+ value: 26.497
+ - type: map_at_3
+ value: 21.61
+ - type: map_at_5
+ value: 23.76
- type: mrr_at_1
- value: 35.179
- - type: mrr_at_2
- value: 41.792
- - type: mrr_at_3
- value: 44.484
- - type: mrr_at_5
- value: 46.39
- - type: mrr_at_7
- value: 47.125
+ value: 34.98371335504886
- type: mrr_at_10
- value: 47.711999999999996
- - type: mrr_at_20
- value: 48.214
- - type: mrr_at_30
- value: 48.325
- - type: mrr_at_50
- value: 48.392
- - type: mrr_at_70
- value: 48.418
+ value: 45.73747479447807
- type: mrr_at_100
- value: 48.44
- - type: mrr_at_200
- value: 48.46
- - type: mrr_at_300
- value: 48.461999999999996
- - type: mrr_at_500
- value: 48.466
- - type: mrr_at_700
- value: 48.466
+ value: 46.4973410206458
- type: mrr_at_1000
- value: 48.467
+ value: 46.53372527933685
+ - type: mrr_at_20
+ value: 46.19978503202757
+ - type: mrr_at_3
+ value: 42.85559174809991
+ - type: mrr_at_5
+ value: 44.65038002171556
+ - type: nauc_map_at_1000_diff1
+ value: 26.351501821886085
+ - type: nauc_map_at_1000_max
+ value: 37.406277027621435
+ - type: nauc_map_at_100_diff1
+ value: 26.323538607543114
+ - type: nauc_map_at_100_max
+ value: 37.37217323049272
+ - type: nauc_map_at_10_diff1
+ value: 26.70038577409459
+ - type: nauc_map_at_10_max
+ value: 37.79383579109463
+ - type: nauc_map_at_1_diff1
+ value: 36.47845928386989
+ - type: nauc_map_at_1_max
+ value: 39.01927463942033
+ - type: nauc_map_at_20_diff1
+ value: 26.33574557573518
+ - type: nauc_map_at_20_max
+ value: 37.137386086830816
+ - type: nauc_map_at_3_diff1
+ value: 28.703462673742525
+ - type: nauc_map_at_3_max
+ value: 39.047906026148546
+ - type: nauc_map_at_5_diff1
+ value: 27.494535996471757
+ - type: nauc_map_at_5_max
+ value: 38.29226067477129
+ - type: nauc_mrr_at_1000_diff1
+ value: 27.021932862086285
+ - type: nauc_mrr_at_1000_max
+ value: 38.48562927637259
+ - type: nauc_mrr_at_100_diff1
+ value: 27.01468202585599
+ - type: nauc_mrr_at_100_max
+ value: 38.48627350276011
+ - type: nauc_mrr_at_10_diff1
+ value: 26.893457244015195
+ - type: nauc_mrr_at_10_max
+ value: 38.55614447790195
+ - type: nauc_mrr_at_1_diff1
+ value: 33.24228449031407
+ - type: nauc_mrr_at_1_max
+ value: 39.88402393762058
+ - type: nauc_mrr_at_20_diff1
+ value: 26.918439332794712
+ - type: nauc_mrr_at_20_max
+ value: 38.4173740673778
+ - type: nauc_mrr_at_3_diff1
+ value: 27.309744542742447
+ - type: nauc_mrr_at_3_max
+ value: 38.582097929189516
+ - type: nauc_mrr_at_5_diff1
+ value: 26.800630213350797
+ - type: nauc_mrr_at_5_max
+ value: 38.188788097352564
+ - type: nauc_ndcg_at_1000_diff1
+ value: 24.327541764652814
+ - type: nauc_ndcg_at_1000_max
+ value: 37.23069926131115
+ - type: nauc_ndcg_at_100_diff1
+ value: 23.8579715304784
+ - type: nauc_ndcg_at_100_max
+ value: 36.508265535016555
+ - type: nauc_ndcg_at_10_diff1
+ value: 24.018925837082865
+ - type: nauc_ndcg_at_10_max
+ value: 36.770343111777315
+ - type: nauc_ndcg_at_1_diff1
+ value: 33.24228449031407
+ - type: nauc_ndcg_at_1_max
+ value: 39.88402393762058
+ - type: nauc_ndcg_at_20_diff1
+ value: 23.436752536514412
+ - type: nauc_ndcg_at_20_max
+ value: 35.403527798009144
+ - type: nauc_ndcg_at_3_diff1
+ value: 26.055015016762802
+ - type: nauc_ndcg_at_3_max
+ value: 38.36860877229367
+ - type: nauc_ndcg_at_5_diff1
+ value: 25.038417711012706
+ - type: nauc_ndcg_at_5_max
+ value: 37.346514402237325
+ - type: nauc_precision_at_1000_diff1
+ value: -5.078817361582684
+ - type: nauc_precision_at_1000_max
+ value: 4.582301323697963
+ - type: nauc_precision_at_100_diff1
+ value: -0.6557129555664837
+ - type: nauc_precision_at_100_max
+ value: 11.77925636740658
+ - type: nauc_precision_at_10_diff1
+ value: 4.943563540918312
+ - type: nauc_precision_at_10_max
+ value: 21.18202054009953
+ - type: nauc_precision_at_1_diff1
+ value: 33.24228449031407
+ - type: nauc_precision_at_1_max
+ value: 39.88402393762058
+ - type: nauc_precision_at_20_diff1
+ value: 1.836926537444962
+ - type: nauc_precision_at_20_max
+ value: 14.040874706477421
+ - type: nauc_precision_at_3_diff1
+ value: 14.266103804270482
+ - type: nauc_precision_at_3_max
+ value: 33.31831003283801
+ - type: nauc_precision_at_5_diff1
+ value: 9.37611184447737
+ - type: nauc_precision_at_5_max
+ value: 26.95142588409704
+ - type: nauc_recall_at_1000_diff1
+ value: 13.892802154238765
+ - type: nauc_recall_at_1000_max
+ value: 28.193901567417818
+ - type: nauc_recall_at_100_diff1
+ value: 14.02284707047029
+ - type: nauc_recall_at_100_max
+ value: 25.700501478581412
+ - type: nauc_recall_at_10_diff1
+ value: 16.09379573467772
+ - type: nauc_recall_at_10_max
+ value: 29.455588696309732
+ - type: nauc_recall_at_1_diff1
+ value: 36.47845928386989
+ - type: nauc_recall_at_1_max
+ value: 39.01927463942033
+ - type: nauc_recall_at_20_diff1
+ value: 14.09153116597221
+ - type: nauc_recall_at_20_max
+ value: 24.880078799144776
+ - type: nauc_recall_at_3_diff1
+ value: 22.70624020195598
+ - type: nauc_recall_at_3_max
+ value: 36.050690466903255
+ - type: nauc_recall_at_5_diff1
+ value: 19.08495953623003
+ - type: nauc_recall_at_5_max
+ value: 32.34052391885703
+ - type: ndcg_at_1
+ value: 34.984
+ - type: ndcg_at_10
+ value: 34.427
+ - type: ndcg_at_100
+ value: 40.908
+ - type: ndcg_at_1000
+ value: 44.118
+ - type: ndcg_at_20
+ value: 36.885
+ - type: ndcg_at_3
+ value: 29.09
+ - type: ndcg_at_5
+ value: 30.979
+ - type: precision_at_1
+ value: 34.984
+ - type: precision_at_10
+ value: 10.476
+ - type: precision_at_100
+ value: 1.748
+ - type: precision_at_1000
+ value: 0.23500000000000001
+ - type: precision_at_20
+ value: 6.313000000000001
+ - type: precision_at_3
+ value: 21.39
+ - type: precision_at_5
+ value: 16.378
+ - type: recall_at_1
+ value: 15.609
+ - type: recall_at_10
+ value: 39.619
+ - type: recall_at_100
+ value: 61.952
+ - type: recall_at_1000
+ value: 79.861
+ - type: recall_at_20
+ value: 46.489000000000004
+ - type: recall_at_3
+ value: 26.134
+ - type: recall_at_5
+ value: 31.955
- task:
type: Retrieval
dataset:
- type: dbpedia-entity
+ type: mteb/dbpedia
name: MTEB DBPedia
config: default
split: test
- revision: None
+ revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659
metrics:
- - type: ndcg_at_1
- value: 62.375
- - type: ndcg_at_2
- value: 56.286
- - type: ndcg_at_3
- value: 53.665
- - type: ndcg_at_5
- value: 51.139
- - type: ndcg_at_7
- value: 49.873
- - type: ndcg_at_10
- value: 49.056
- - type: ndcg_at_20
- value: 48.783
- - type: ndcg_at_30
- value: 49.166
- - type: ndcg_at_50
- value: 51.141999999999996
- - type: ndcg_at_70
- value: 52.774
- - type: ndcg_at_100
- value: 54.403
- - type: ndcg_at_200
- value: 57.419
- - type: ndcg_at_300
- value: 58.778
- - type: ndcg_at_500
- value: 60.228
- - type: ndcg_at_700
- value: 61.07599999999999
- - type: ndcg_at_1000
- value: 61.846000000000004
- type: map_at_1
- value: 10.359
- - type: map_at_2
- value: 14.446
- - type: map_at_3
- value: 16.689
- - type: map_at_5
- value: 20.096
- - type: map_at_7
- value: 22.247
+ value: 10.482
- type: map_at_10
- value: 24.468999999999998
- - type: map_at_20
- value: 28.938000000000002
- - type: map_at_30
- value: 31.134
- - type: map_at_50
- value: 33.403
- - type: map_at_70
- value: 34.486
+ value: 25.155
- type: map_at_100
- value: 35.337
- - type: map_at_200
- value: 36.364999999999995
- - type: map_at_300
- value: 36.735
- - type: map_at_500
- value: 37.057
- - type: map_at_700
- value: 37.225
+ value: 36.606
- type: map_at_1000
- value: 37.379
- - type: recall_at_1
- value: 10.359
- - type: recall_at_2
- value: 14.945
- - type: recall_at_3
- value: 17.694
- - type: recall_at_5
- value: 22.677
- - type: recall_at_7
- value: 26.131
- - type: recall_at_10
- value: 30.053
- - type: recall_at_20
- value: 39.518
- - type: recall_at_30
- value: 44.925
- - type: recall_at_50
- value: 52.154
- - type: recall_at_70
- value: 56.729
- - type: recall_at_100
- value: 61.18900000000001
- - type: recall_at_200
- value: 70.407
- - type: recall_at_300
- value: 74.412
- - type: recall_at_500
- value: 78.891
- - type: recall_at_700
- value: 81.74
- - type: recall_at_1000
- value: 84.253
- - type: precision_at_1
- value: 75
- - type: precision_at_2
- value: 64.125
- - type: precision_at_3
- value: 57.833
- - type: precision_at_5
- value: 50.24999999999999
- - type: precision_at_7
- value: 44.75
- - type: precision_at_10
- value: 39.75
- - type: precision_at_20
- value: 30.412
- - type: precision_at_30
- value: 25.141999999999996
- - type: precision_at_50
- value: 19.2
- - type: precision_at_70
- value: 15.729000000000001
- - type: precision_at_100
- value: 12.552
- - type: precision_at_200
- value: 7.866
- - type: precision_at_300
- value: 5.9270000000000005
- - type: precision_at_500
- value: 4.1129999999999995
- - type: precision_at_700
- value: 3.2460000000000004
- - type: precision_at_1000
- value: 2.5260000000000002
+ value: 38.617000000000004
+ - type: map_at_20
+ value: 29.676000000000002
+ - type: map_at_3
+ value: 16.881
+ - type: map_at_5
+ value: 20.043
- type: mrr_at_1
- value: 75
- - type: mrr_at_2
- value: 78.625
- - type: mrr_at_3
- value: 79.708
- - type: mrr_at_5
- value: 80.446
- - type: mrr_at_7
- value: 80.862
+ value: 76.0
- type: mrr_at_10
- value: 81.161
- - type: mrr_at_20
- value: 81.3
- - type: mrr_at_30
- value: 81.348
- - type: mrr_at_50
- value: 81.361
- - type: mrr_at_70
- value: 81.361
+ value: 82.5610119047619
- type: mrr_at_100
- value: 81.361
- - type: mrr_at_200
- value: 81.367
- - type: mrr_at_300
- value: 81.367
- - type: mrr_at_500
- value: 81.368
- - type: mrr_at_700
- value: 81.368
+ value: 82.74795937825128
- type: mrr_at_1000
- value: 81.368
+ value: 82.75526942226163
+ - type: mrr_at_20
+ value: 82.70580357142858
+ - type: mrr_at_3
+ value: 81.41666666666667
+ - type: mrr_at_5
+ value: 82.26666666666667
+ - type: nauc_map_at_1000_diff1
+ value: 21.445110500859467
+ - type: nauc_map_at_1000_max
+ value: 31.995436955171268
+ - type: nauc_map_at_100_diff1
+ value: 22.08580365858692
+ - type: nauc_map_at_100_max
+ value: 30.87564480562495
+ - type: nauc_map_at_10_diff1
+ value: 23.200721963841133
+ - type: nauc_map_at_10_max
+ value: 14.351516406083103
+ - type: nauc_map_at_1_diff1
+ value: 32.52519501378748
+ - type: nauc_map_at_1_max
+ value: 1.4652199942711484
+ - type: nauc_map_at_20_diff1
+ value: 22.562447783667206
+ - type: nauc_map_at_20_max
+ value: 21.010455427299586
+ - type: nauc_map_at_3_diff1
+ value: 26.93913513039819
+ - type: nauc_map_at_3_max
+ value: 7.461464428866808
+ - type: nauc_map_at_5_diff1
+ value: 24.91575143410333
+ - type: nauc_map_at_5_max
+ value: 9.94360484450561
+ - type: nauc_mrr_at_1000_diff1
+ value: 59.421296549180354
+ - type: nauc_mrr_at_1000_max
+ value: 66.74548608831367
+ - type: nauc_mrr_at_100_diff1
+ value: 59.40522345017357
+ - type: nauc_mrr_at_100_max
+ value: 66.74283926838633
+ - type: nauc_mrr_at_10_diff1
+ value: 59.076224687978815
+ - type: nauc_mrr_at_10_max
+ value: 66.65665083298556
+ - type: nauc_mrr_at_1_diff1
+ value: 64.7264644804015
+ - type: nauc_mrr_at_1_max
+ value: 65.38623777797005
+ - type: nauc_mrr_at_20_diff1
+ value: 59.40899252002873
+ - type: nauc_mrr_at_20_max
+ value: 66.74964758956885
+ - type: nauc_mrr_at_3_diff1
+ value: 59.51240294969479
+ - type: nauc_mrr_at_3_max
+ value: 67.16855592771704
+ - type: nauc_mrr_at_5_diff1
+ value: 58.89749424787042
+ - type: nauc_mrr_at_5_max
+ value: 67.0342865886284
+ - type: nauc_ndcg_at_1000_diff1
+ value: 31.71920829799219
+ - type: nauc_ndcg_at_1000_max
+ value: 48.74925851197978
+ - type: nauc_ndcg_at_100_diff1
+ value: 31.580377214577187
+ - type: nauc_ndcg_at_100_max
+ value: 44.60890109420616
+ - type: nauc_ndcg_at_10_diff1
+ value: 28.433242953104237
+ - type: nauc_ndcg_at_10_max
+ value: 42.31080688368063
+ - type: nauc_ndcg_at_1_diff1
+ value: 52.820101521576944
+ - type: nauc_ndcg_at_1_max
+ value: 51.17718070882259
+ - type: nauc_ndcg_at_20_diff1
+ value: 29.751022022812002
+ - type: nauc_ndcg_at_20_max
+ value: 40.00186139991905
+ - type: nauc_ndcg_at_3_diff1
+ value: 32.40862612934291
+ - type: nauc_ndcg_at_3_max
+ value: 46.1692618192983
+ - type: nauc_ndcg_at_5_diff1
+ value: 28.672960301655472
+ - type: nauc_ndcg_at_5_max
+ value: 44.941890156982865
+ - type: nauc_precision_at_1000_diff1
+ value: -20.99374169748155
+ - type: nauc_precision_at_1000_max
+ value: -6.577124089078604
+ - type: nauc_precision_at_100_diff1
+ value: -8.142460575139692
+ - type: nauc_precision_at_100_max
+ value: 23.277294094084674
+ - type: nauc_precision_at_10_diff1
+ value: -4.92238102391072
+ - type: nauc_precision_at_10_max
+ value: 33.044818739342276
+ - type: nauc_precision_at_1_diff1
+ value: 64.7264644804015
+ - type: nauc_precision_at_1_max
+ value: 65.38623777797005
+ - type: nauc_precision_at_20_diff1
+ value: -5.003581741495679
+ - type: nauc_precision_at_20_max
+ value: 31.864078466638496
+ - type: nauc_precision_at_3_diff1
+ value: 12.977896117859569
+ - type: nauc_precision_at_3_max
+ value: 45.22814965056419
+ - type: nauc_precision_at_5_diff1
+ value: 1.15598919880241
+ - type: nauc_precision_at_5_max
+ value: 37.81588472196163
+ - type: nauc_recall_at_1000_diff1
+ value: 25.43067172374462
+ - type: nauc_recall_at_1000_max
+ value: 44.57570549352574
+ - type: nauc_recall_at_100_diff1
+ value: 22.4908346680046
+ - type: nauc_recall_at_100_max
+ value: 32.72066413483283
+ - type: nauc_recall_at_10_diff1
+ value: 17.71571038757097
+ - type: nauc_recall_at_10_max
+ value: 8.31205267676276
+ - type: nauc_recall_at_1_diff1
+ value: 32.52519501378748
+ - type: nauc_recall_at_1_max
+ value: 1.4652199942711484
+ - type: nauc_recall_at_20_diff1
+ value: 18.729697207082424
+ - type: nauc_recall_at_20_max
+ value: 15.146501158005584
+ - type: nauc_recall_at_3_diff1
+ value: 23.235299623059735
+ - type: nauc_recall_at_3_max
+ value: 5.513207542737543
+ - type: nauc_recall_at_5_diff1
+ value: 19.279237075855654
+ - type: nauc_recall_at_5_max
+ value: 6.108691272763718
+ - type: ndcg_at_1
+ value: 63.625
+ - type: ndcg_at_10
+ value: 51.214000000000006
+ - type: ndcg_at_100
+ value: 56.411
+ - type: ndcg_at_1000
+ value: 63.429
+ - type: ndcg_at_20
+ value: 50.595
+ - type: ndcg_at_3
+ value: 54.989
+ - type: ndcg_at_5
+ value: 52.589
+ - type: precision_at_1
+ value: 76.0
+ - type: precision_at_10
+ value: 41.975
+ - type: precision_at_100
+ value: 13.26
+ - type: precision_at_1000
+ value: 2.493
+ - type: precision_at_20
+ value: 32.15
+ - type: precision_at_3
+ value: 59.0
+ - type: precision_at_5
+ value: 51.24999999999999
+ - type: recall_at_1
+ value: 10.482
+ - type: recall_at_10
+ value: 31.075000000000003
+ - type: recall_at_100
+ value: 63.119
+ - type: recall_at_1000
+ value: 85.32300000000001
+ - type: recall_at_20
+ value: 40.345
+ - type: recall_at_3
+ value: 17.916
+ - type: recall_at_5
+ value: 22.475
- task:
type: Classification
dataset:
@@ -826,516 +1032,666 @@ model-index:
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37
metrics:
- type: accuracy
- value: 50.239999999999995
+ value: 93.36500000000001
- type: f1
- value: 46.42361822342044
+ value: 89.89541440183861
- task:
type: Retrieval
dataset:
- type: fever
+ type: mteb/fever
name: MTEB FEVER
config: default
split: test
- revision: None
+ revision: bea83ef9e8fb933d90a2f1d5515737465d613e12
metrics:
- - type: ndcg_at_1
- value: 83.723
- - type: ndcg_at_2
- value: 86.777
- - type: ndcg_at_3
- value: 87.997
- - type: ndcg_at_5
- value: 88.864
- - type: ndcg_at_7
- value: 89.143
- - type: ndcg_at_10
- value: 89.349
- - type: ndcg_at_20
- value: 89.709
- - type: ndcg_at_30
- value: 89.82900000000001
- - type: ndcg_at_50
- value: 89.923
- - type: ndcg_at_70
- value: 89.982
- - type: ndcg_at_100
- value: 90.026
- - type: ndcg_at_200
- value: 90.10000000000001
- - type: ndcg_at_300
- value: 90.12599999999999
- - type: ndcg_at_500
- value: 90.17399999999999
- - type: ndcg_at_700
- value: 90.19
- - type: ndcg_at_1000
- value: 90.208
- type: map_at_1
- value: 77.64999999999999
- - type: map_at_2
- value: 83.769
- - type: map_at_3
- value: 85.041
- - type: map_at_5
- value: 85.736
- - type: map_at_7
- value: 85.924
+ value: 81.948
- type: map_at_10
- value: 86.032
- - type: map_at_20
- value: 86.177
- - type: map_at_30
- value: 86.213
- - type: map_at_50
- value: 86.233
- - type: map_at_70
- value: 86.24300000000001
+ value: 89.47500000000001
- type: map_at_100
- value: 86.249
- - type: map_at_200
- value: 86.256
- - type: map_at_300
- value: 86.258
- - type: map_at_500
- value: 86.26
- - type: map_at_700
- value: 86.26
+ value: 89.66199999999999
- type: map_at_1000
- value: 86.261
- - type: recall_at_1
- value: 77.64999999999999
- - type: recall_at_2
- value: 88.53999999999999
- - type: recall_at_3
- value: 91.696
- - type: recall_at_5
- value: 93.916
- - type: recall_at_7
- value: 94.731
- - type: recall_at_10
- value: 95.318
- - type: recall_at_20
- value: 96.507
- - type: recall_at_30
- value: 96.956
- - type: recall_at_50
- value: 97.34899999999999
- - type: recall_at_70
- value: 97.61
- - type: recall_at_100
- value: 97.83
- - type: recall_at_200
- value: 98.223
- - type: recall_at_300
- value: 98.374
- - type: recall_at_500
- value: 98.67899999999999
- - type: recall_at_700
- value: 98.787
- - type: recall_at_1000
- value: 98.919
- - type: precision_at_1
- value: 83.723
- - type: precision_at_2
- value: 48.425000000000004
- - type: precision_at_3
- value: 33.638
- - type: precision_at_5
- value: 20.843
- - type: precision_at_7
- value: 15.079
- - type: precision_at_10
- value: 10.674999999999999
- - type: precision_at_20
- value: 5.457999999999999
- - type: precision_at_30
- value: 3.6740000000000004
- - type: precision_at_50
- value: 2.2239999999999998
- - type: precision_at_70
- value: 1.599
- - type: precision_at_100
- value: 1.125
- - type: precision_at_200
- value: 0.5680000000000001
- - type: precision_at_300
- value: 0.38
- - type: precision_at_500
- value: 0.22999999999999998
- - type: precision_at_700
- value: 0.165
- - type: precision_at_1000
- value: 0.116
+ value: 89.671
+ - type: map_at_20
+ value: 89.582
+ - type: map_at_3
+ value: 88.646
+ - type: map_at_5
+ value: 89.19
- type: mrr_at_1
- value: 83.723
- - type: mrr_at_2
- value: 88.794
- - type: mrr_at_3
- value: 89.679
- - type: mrr_at_5
- value: 90.049
- - type: mrr_at_7
- value: 90.129
+ value: 88.23882388238825
- type: mrr_at_10
- value: 90.167
- - type: mrr_at_20
- value: 90.208
- - type: mrr_at_30
- value: 90.214
- - type: mrr_at_50
- value: 90.217
- - type: mrr_at_70
- value: 90.218
+ value: 93.2122736083131
- type: mrr_at_100
- value: 90.21900000000001
- - type: mrr_at_200
- value: 90.21900000000001
- - type: mrr_at_300
- value: 90.21900000000001
- - type: mrr_at_500
- value: 90.21900000000001
- - type: mrr_at_700
- value: 90.21900000000001
+ value: 93.23908769526588
- type: mrr_at_1000
- value: 90.21900000000001
+ value: 93.23932393435209
+ - type: mrr_at_20
+ value: 93.23217832106207
+ - type: mrr_at_3
+ value: 92.98679867986787
+ - type: mrr_at_5
+ value: 93.16906690669056
+ - type: nauc_map_at_1000_diff1
+ value: 52.317305666990336
+ - type: nauc_map_at_1000_max
+ value: 24.679818502783885
+ - type: nauc_map_at_100_diff1
+ value: 52.25337722551868
+ - type: nauc_map_at_100_max
+ value: 24.660099692702346
+ - type: nauc_map_at_10_diff1
+ value: 51.613988585217804
+ - type: nauc_map_at_10_max
+ value: 24.27239106759732
+ - type: nauc_map_at_1_diff1
+ value: 59.07534243064022
+ - type: nauc_map_at_1_max
+ value: 21.60122433075861
+ - type: nauc_map_at_20_diff1
+ value: 51.89863964441336
+ - type: nauc_map_at_20_max
+ value: 24.43166858219192
+ - type: nauc_map_at_3_diff1
+ value: 50.59233576530098
+ - type: nauc_map_at_3_max
+ value: 24.12500680105456
+ - type: nauc_map_at_5_diff1
+ value: 50.8925873150748
+ - type: nauc_map_at_5_max
+ value: 24.07083932732526
+ - type: nauc_mrr_at_1000_diff1
+ value: 75.93025830228439
+ - type: nauc_mrr_at_1000_max
+ value: 30.19272028005147
+ - type: nauc_mrr_at_100_diff1
+ value: 75.92956741572067
+ - type: nauc_mrr_at_100_max
+ value: 30.194676664511515
+ - type: nauc_mrr_at_10_diff1
+ value: 75.89011155692661
+ - type: nauc_mrr_at_10_max
+ value: 30.37775880877866
+ - type: nauc_mrr_at_1_diff1
+ value: 77.44260299645404
+ - type: nauc_mrr_at_1_max
+ value: 26.559431934521747
+ - type: nauc_mrr_at_20_diff1
+ value: 75.891450647917
+ - type: nauc_mrr_at_20_max
+ value: 30.21977375474477
+ - type: nauc_mrr_at_3_diff1
+ value: 75.6014258246174
+ - type: nauc_mrr_at_3_max
+ value: 30.962934137235816
+ - type: nauc_mrr_at_5_diff1
+ value: 75.81100281536122
+ - type: nauc_mrr_at_5_max
+ value: 30.61294373818605
+ - type: nauc_ndcg_at_1000_diff1
+ value: 55.88212949654192
+ - type: nauc_ndcg_at_1000_max
+ value: 26.654306655604792
+ - type: nauc_ndcg_at_100_diff1
+ value: 54.573720387585844
+ - type: nauc_ndcg_at_100_max
+ value: 26.40693679116972
+ - type: nauc_ndcg_at_10_diff1
+ value: 51.88967519133213
+ - type: nauc_ndcg_at_10_max
+ value: 25.570560464465025
+ - type: nauc_ndcg_at_1_diff1
+ value: 77.44260299645404
+ - type: nauc_ndcg_at_1_max
+ value: 26.559431934521747
+ - type: nauc_ndcg_at_20_diff1
+ value: 52.58682804821843
+ - type: nauc_ndcg_at_20_max
+ value: 25.696982642067052
+ - type: nauc_ndcg_at_3_diff1
+ value: 52.243687096362976
+ - type: nauc_ndcg_at_3_max
+ value: 26.984804879189934
+ - type: nauc_ndcg_at_5_diff1
+ value: 51.21265931992794
+ - type: nauc_ndcg_at_5_max
+ value: 25.916035266211697
+ - type: nauc_precision_at_1000_diff1
+ value: -7.90458882177324
+ - type: nauc_precision_at_1000_max
+ value: -0.919981996545353
+ - type: nauc_precision_at_100_diff1
+ value: -11.033772404215371
+ - type: nauc_precision_at_100_max
+ value: -1.055247613217208
+ - type: nauc_precision_at_10_diff1
+ value: -14.785542578940273
+ - type: nauc_precision_at_10_max
+ value: -1.9557491629741635
+ - type: nauc_precision_at_1_diff1
+ value: 77.44260299645404
+ - type: nauc_precision_at_1_max
+ value: 26.559431934521747
+ - type: nauc_precision_at_20_diff1
+ value: -14.428848368307829
+ - type: nauc_precision_at_20_max
+ value: -1.9241265648483015
+ - type: nauc_precision_at_3_diff1
+ value: -10.406952580943814
+ - type: nauc_precision_at_3_max
+ value: 3.5903711345349776
+ - type: nauc_precision_at_5_diff1
+ value: -15.451415653007858
+ - type: nauc_precision_at_5_max
+ value: -0.7459331254150773
+ - type: nauc_recall_at_1000_diff1
+ value: 10.73273725772606
+ - type: nauc_recall_at_1000_max
+ value: 36.40274799382754
+ - type: nauc_recall_at_100_diff1
+ value: -2.162087994352105
+ - type: nauc_recall_at_100_max
+ value: 26.473789390365777
+ - type: nauc_recall_at_10_diff1
+ value: 11.644894941881944
+ - type: nauc_recall_at_10_max
+ value: 23.14164891130646
+ - type: nauc_recall_at_1_diff1
+ value: 59.07534243064022
+ - type: nauc_recall_at_1_max
+ value: 21.60122433075861
+ - type: nauc_recall_at_20_diff1
+ value: 5.566916083163112
+ - type: nauc_recall_at_20_max
+ value: 21.86933138525287
+ - type: nauc_recall_at_3_diff1
+ value: 23.81750212101455
+ - type: nauc_recall_at_3_max
+ value: 24.980128585581422
+ - type: nauc_recall_at_5_diff1
+ value: 16.069208420506715
+ - type: nauc_recall_at_5_max
+ value: 23.89156092743343
+ - type: ndcg_at_1
+ value: 88.239
+ - type: ndcg_at_10
+ value: 92.155
+ - type: ndcg_at_100
+ value: 92.735
+ - type: ndcg_at_1000
+ value: 92.866
+ - type: ndcg_at_20
+ value: 92.39699999999999
+ - type: ndcg_at_3
+ value: 91.188
+ - type: ndcg_at_5
+ value: 91.754
+ - type: precision_at_1
+ value: 88.239
+ - type: precision_at_10
+ value: 10.903
+ - type: precision_at_100
+ value: 1.147
+ - type: precision_at_1000
+ value: 0.117
+ - type: precision_at_20
+ value: 5.5440000000000005
+ - type: precision_at_3
+ value: 34.598
+ - type: precision_at_5
+ value: 21.302
+ - type: recall_at_1
+ value: 81.948
+ - type: recall_at_10
+ value: 96.518
+ - type: recall_at_100
+ value: 98.646
+ - type: recall_at_1000
+ value: 99.399
+ - type: recall_at_20
+ value: 97.262
+ - type: recall_at_3
+ value: 93.89800000000001
+ - type: recall_at_5
+ value: 95.38600000000001
- task:
type: Retrieval
dataset:
- type: fiqa
+ type: mteb/fiqa
name: MTEB FiQA2018
config: default
split: test
- revision: None
+ revision: 27a168819829fe9bcd655c2df245fb19452e8e06
metrics:
- - type: ndcg_at_1
- value: 59.721999999999994
- - type: ndcg_at_2
- value: 56.85
- - type: ndcg_at_3
- value: 56.462999999999994
- - type: ndcg_at_5
- value: 57.75599999999999
- - type: ndcg_at_7
- value: 59.109
- - type: ndcg_at_10
- value: 60.402
- - type: ndcg_at_20
- value: 63.071999999999996
- - type: ndcg_at_30
- value: 64.302
- - type: ndcg_at_50
- value: 65.619
- - type: ndcg_at_70
- value: 66.161
- - type: ndcg_at_100
- value: 66.645
- - type: ndcg_at_200
- value: 67.353
- - type: ndcg_at_300
- value: 67.646
- - type: ndcg_at_500
- value: 67.852
- - type: ndcg_at_700
- value: 67.974
- - type: ndcg_at_1000
- value: 68.084
- type: map_at_1
- value: 31.56
- - type: map_at_2
- value: 42.093
- - type: map_at_3
- value: 46.177
- - type: map_at_5
- value: 49.78
- - type: map_at_7
- value: 51.410999999999994
+ value: 32.033
- type: map_at_10
- value: 52.524
- - type: map_at_20
- value: 53.815000000000005
- - type: map_at_30
- value: 54.201
- - type: map_at_50
- value: 54.531
- - type: map_at_70
- value: 54.625
+ value: 53.55
- type: map_at_100
- value: 54.686
- - type: map_at_200
- value: 54.757999999999996
- - type: map_at_300
- value: 54.776
- - type: map_at_500
- value: 54.786
- - type: map_at_700
- value: 54.790000000000006
+ value: 55.672
- type: map_at_1000
- value: 54.793000000000006
- - type: recall_at_1
- value: 31.56
- - type: recall_at_2
- value: 44.858
- - type: recall_at_3
- value: 51.11
- - type: recall_at_5
- value: 58.394
- - type: recall_at_7
- value: 63.001
- - type: recall_at_10
- value: 66.81200000000001
- - type: recall_at_20
- value: 74.901
- - type: recall_at_30
- value: 79.218
- - type: recall_at_50
- value: 84.49
- - type: recall_at_70
- value: 87.003
- - type: recall_at_100
- value: 89.345
- - type: recall_at_200
- value: 93.173
- - type: recall_at_300
- value: 94.906
- - type: recall_at_500
- value: 96.223
- - type: recall_at_700
- value: 97.043
- - type: recall_at_1000
- value: 97.785
+ value: 55.764
+ - type: map_at_20
+ value: 54.87800000000001
+ - type: map_at_3
+ value: 46.761
+ - type: map_at_5
+ value: 50.529
+ - type: mrr_at_1
+ value: 60.95679012345679
+ - type: mrr_at_10
+ value: 68.70835782872815
+ - type: mrr_at_100
+ value: 69.21918402444501
+ - type: mrr_at_1000
+ value: 69.23608783148705
+ - type: mrr_at_20
+ value: 69.07497388036454
+ - type: mrr_at_3
+ value: 66.76954732510285
+ - type: mrr_at_5
+ value: 67.95781893004109
+ - type: nauc_map_at_1000_diff1
+ value: 50.29193951970748
+ - type: nauc_map_at_1000_max
+ value: 47.19671414418149
+ - type: nauc_map_at_100_diff1
+ value: 50.297759303868474
+ - type: nauc_map_at_100_max
+ value: 47.19481602426411
+ - type: nauc_map_at_10_diff1
+ value: 50.62894041657141
+ - type: nauc_map_at_10_max
+ value: 45.567759689277395
+ - type: nauc_map_at_1_diff1
+ value: 53.59982673587883
+ - type: nauc_map_at_1_max
+ value: 25.977384172159706
+ - type: nauc_map_at_20_diff1
+ value: 50.27749492520328
+ - type: nauc_map_at_20_max
+ value: 46.64504183104472
+ - type: nauc_map_at_3_diff1
+ value: 50.615957546204335
+ - type: nauc_map_at_3_max
+ value: 37.81289784841956
+ - type: nauc_map_at_5_diff1
+ value: 50.928252694106554
+ - type: nauc_map_at_5_max
+ value: 42.49688254915941
+ - type: nauc_mrr_at_1000_diff1
+ value: 61.65261451863071
+ - type: nauc_mrr_at_1000_max
+ value: 58.65098270465221
+ - type: nauc_mrr_at_100_diff1
+ value: 61.62990859158191
+ - type: nauc_mrr_at_100_max
+ value: 58.65624372299693
+ - type: nauc_mrr_at_10_diff1
+ value: 61.74994032364257
+ - type: nauc_mrr_at_10_max
+ value: 58.62854864341972
+ - type: nauc_mrr_at_1_diff1
+ value: 64.71262154210315
+ - type: nauc_mrr_at_1_max
+ value: 58.70327874526542
+ - type: nauc_mrr_at_20_diff1
+ value: 61.58111345058981
+ - type: nauc_mrr_at_20_max
+ value: 58.63734626996976
+ - type: nauc_mrr_at_3_diff1
+ value: 61.36455839525452
+ - type: nauc_mrr_at_3_max
+ value: 58.43927781578019
+ - type: nauc_mrr_at_5_diff1
+ value: 61.943728083423764
+ - type: nauc_mrr_at_5_max
+ value: 58.88792355920571
+ - type: nauc_ndcg_at_1000_diff1
+ value: 52.423665123234166
+ - type: nauc_ndcg_at_1000_max
+ value: 52.240649255471894
+ - type: nauc_ndcg_at_100_diff1
+ value: 51.960274190332335
+ - type: nauc_ndcg_at_100_max
+ value: 52.486936655426085
+ - type: nauc_ndcg_at_10_diff1
+ value: 52.17693285192987
+ - type: nauc_ndcg_at_10_max
+ value: 49.19718343617846
+ - type: nauc_ndcg_at_1_diff1
+ value: 64.71262154210315
+ - type: nauc_ndcg_at_1_max
+ value: 58.70327874526542
+ - type: nauc_ndcg_at_20_diff1
+ value: 51.30403706422037
+ - type: nauc_ndcg_at_20_max
+ value: 50.46377359818194
+ - type: nauc_ndcg_at_3_diff1
+ value: 48.841478058787054
+ - type: nauc_ndcg_at_3_max
+ value: 48.73801532909532
+ - type: nauc_ndcg_at_5_diff1
+ value: 51.332087141067184
+ - type: nauc_ndcg_at_5_max
+ value: 48.52472774603273
+ - type: nauc_precision_at_1000_diff1
+ value: -15.772766658041673
+ - type: nauc_precision_at_1000_max
+ value: 21.177705914829428
+ - type: nauc_precision_at_100_diff1
+ value: -11.555646100921061
+ - type: nauc_precision_at_100_max
+ value: 27.832707769630403
+ - type: nauc_precision_at_10_diff1
+ value: 4.165465749146322
+ - type: nauc_precision_at_10_max
+ value: 40.0518003689246
+ - type: nauc_precision_at_1_diff1
+ value: 64.71262154210315
+ - type: nauc_precision_at_1_max
+ value: 58.70327874526542
+ - type: nauc_precision_at_20_diff1
+ value: -3.3255458383587446
+ - type: nauc_precision_at_20_max
+ value: 35.42291884694886
+ - type: nauc_precision_at_3_diff1
+ value: 19.697810607681262
+ - type: nauc_precision_at_3_max
+ value: 45.49968293476392
+ - type: nauc_precision_at_5_diff1
+ value: 12.867812160931885
+ - type: nauc_precision_at_5_max
+ value: 44.278925413285044
+ - type: nauc_recall_at_1000_diff1
+ value: 41.950516559930676
+ - type: nauc_recall_at_1000_max
+ value: 50.313202372557484
+ - type: nauc_recall_at_100_diff1
+ value: 33.744111523885
+ - type: nauc_recall_at_100_max
+ value: 51.44698437343768
+ - type: nauc_recall_at_10_diff1
+ value: 42.21394194713513
+ - type: nauc_recall_at_10_max
+ value: 38.800282673459456
+ - type: nauc_recall_at_1_diff1
+ value: 53.59982673587883
+ - type: nauc_recall_at_1_max
+ value: 25.977384172159706
+ - type: nauc_recall_at_20_diff1
+ value: 36.106289648299956
+ - type: nauc_recall_at_20_max
+ value: 40.310801276824996
+ - type: nauc_recall_at_3_diff1
+ value: 43.92657693343522
+ - type: nauc_recall_at_3_max
+ value: 31.53598363228886
+ - type: nauc_recall_at_5_diff1
+ value: 44.576198408543476
+ - type: nauc_recall_at_5_max
+ value: 36.8212420206528
+ - type: ndcg_at_1
+ value: 60.956999999999994
+ - type: ndcg_at_10
+ value: 61.766
+ - type: ndcg_at_100
+ value: 67.652
+ - type: ndcg_at_1000
+ value: 68.94500000000001
+ - type: ndcg_at_20
+ value: 64.48700000000001
+ - type: ndcg_at_3
+ value: 57.25
+ - type: ndcg_at_5
+ value: 58.64
- type: precision_at_1
- value: 59.721999999999994
- - type: precision_at_2
- value: 46.682
- - type: precision_at_3
- value: 37.602999999999994
- - type: precision_at_5
- value: 27.500000000000004
- - type: precision_at_7
- value: 21.847
+ value: 60.956999999999994
- type: precision_at_10
- value: 16.667
- - type: precision_at_20
- value: 9.545
- - type: precision_at_30
- value: 6.795
- - type: precision_at_50
- value: 4.38
- - type: precision_at_70
- value: 3.221
+ value: 17.083000000000002
- type: precision_at_100
- value: 2.319
- - type: precision_at_200
- value: 1.2149999999999999
- - type: precision_at_300
- value: 0.827
- - type: precision_at_500
- value: 0.504
- - type: precision_at_700
- value: 0.364
+ value: 2.346
- type: precision_at_1000
value: 0.257
- - type: mrr_at_1
- value: 59.721999999999994
- - type: mrr_at_2
- value: 64.506
- - type: mrr_at_3
- value: 65.792
- - type: mrr_at_5
- value: 66.965
- - type: mrr_at_7
- value: 67.34700000000001
- - type: mrr_at_10
- value: 67.57
- - type: mrr_at_20
- value: 67.896
- - type: mrr_at_30
- value: 68.008
- - type: mrr_at_50
- value: 68.083
- - type: mrr_at_70
- value: 68.105
- - type: mrr_at_100
- value: 68.116
- - type: mrr_at_200
- value: 68.12700000000001
- - type: mrr_at_300
- value: 68.13
- - type: mrr_at_500
- value: 68.132
- - type: mrr_at_700
- value: 68.133
- - type: mrr_at_1000
- value: 68.133
+ - type: precision_at_20
+ value: 9.807
+ - type: precision_at_3
+ value: 38.477
+ - type: precision_at_5
+ value: 27.962999999999997
+ - type: recall_at_1
+ value: 32.033
+ - type: recall_at_10
+ value: 69.44
+ - type: recall_at_100
+ value: 90.17500000000001
+ - type: recall_at_1000
+ value: 97.90100000000001
+ - type: recall_at_20
+ value: 77.629
+ - type: recall_at_3
+ value: 51.664
+ - type: recall_at_5
+ value: 59.565
- task:
type: Retrieval
dataset:
- type: hotpotqa
+ type: mteb/hotpotqa
name: MTEB HotpotQA
config: default
split: test
- revision: None
+ revision: ab518f4d6fcca38d87c25209f94beba119d02014
metrics:
- - type: ndcg_at_1
- value: 81.796
- - type: ndcg_at_2
- value: 67.999
- - type: ndcg_at_3
- value: 72.15599999999999
- - type: ndcg_at_5
- value: 74.99900000000001
- - type: ndcg_at_7
- value: 76.179
- - type: ndcg_at_10
- value: 77.022
- - type: ndcg_at_20
- value: 78.173
- - type: ndcg_at_30
- value: 78.648
- - type: ndcg_at_50
- value: 79.104
- - type: ndcg_at_70
- value: 79.335
- - type: ndcg_at_100
- value: 79.56
- - type: ndcg_at_200
- value: 79.911
- - type: ndcg_at_300
- value: 80.045
- - type: ndcg_at_500
- value: 80.19500000000001
- - type: ndcg_at_700
- value: 80.281
- - type: ndcg_at_1000
- value: 80.35
- type: map_at_1
- value: 40.898
- - type: map_at_2
- value: 62.016000000000005
- - type: map_at_3
- value: 66.121
- - type: map_at_5
- value: 68.471
- - type: map_at_7
- value: 69.261
+ value: 42.741
- type: map_at_10
- value: 69.738
- - type: map_at_20
- value: 70.208
- - type: map_at_30
- value: 70.343
- - type: map_at_50
- value: 70.43700000000001
- - type: map_at_70
- value: 70.47099999999999
+ value: 74.811
- type: map_at_100
- value: 70.498
- - type: map_at_200
- value: 70.526
- - type: map_at_300
- value: 70.533
- - type: map_at_500
- value: 70.538
- - type: map_at_700
- value: 70.541
+ value: 75.508
- type: map_at_1000
- value: 70.542
- - type: recall_at_1
- value: 40.898
- - type: recall_at_2
- value: 63.964
- - type: recall_at_3
- value: 70.743
- - type: recall_at_5
- value: 76.36699999999999
- - type: recall_at_7
- value: 79.142
- - type: recall_at_10
- value: 81.404
- - type: recall_at_20
- value: 85.111
- - type: recall_at_30
- value: 86.92800000000001
- - type: recall_at_50
- value: 88.899
- - type: recall_at_70
- value: 90.01400000000001
- - type: recall_at_100
- value: 91.19500000000001
- - type: recall_at_200
- value: 93.234
- - type: recall_at_300
- value: 94.105
- - type: recall_at_500
- value: 95.159
- - type: recall_at_700
- value: 95.8
- - type: recall_at_1000
- value: 96.34700000000001
- - type: precision_at_1
- value: 81.796
- - type: precision_at_2
- value: 63.964
- - type: precision_at_3
- value: 47.162
- - type: precision_at_5
- value: 30.547
- - type: precision_at_7
- value: 22.612
- - type: precision_at_10
- value: 16.281000000000002
- - type: precision_at_20
- value: 8.511000000000001
- - type: precision_at_30
- value: 5.795
- - type: precision_at_50
- value: 3.556
- - type: precision_at_70
- value: 2.572
- - type: precision_at_100
- value: 1.8239999999999998
- - type: precision_at_200
- value: 0.932
- - type: precision_at_300
- value: 0.627
- - type: precision_at_500
- value: 0.381
- - type: precision_at_700
- value: 0.27399999999999997
- - type: precision_at_1000
- value: 0.193
+ value: 75.541
+ - type: map_at_20
+ value: 75.25699999999999
+ - type: map_at_3
+ value: 71.31
+ - type: map_at_5
+ value: 73.69
- type: mrr_at_1
- value: 81.796
- - type: mrr_at_2
- value: 85.69200000000001
- - type: mrr_at_3
- value: 86.52
- - type: mrr_at_5
- value: 86.973
- - type: mrr_at_7
- value: 87.13300000000001
+ value: 85.48278190411884
- type: mrr_at_10
- value: 87.208
- - type: mrr_at_20
- value: 87.303
- - type: mrr_at_30
- value: 87.32799999999999
- - type: mrr_at_50
- value: 87.347
- - type: mrr_at_70
- value: 87.35199999999999
+ value: 90.20347684425987
- type: mrr_at_100
- value: 87.355
- - type: mrr_at_200
- value: 87.357
- - type: mrr_at_300
- value: 87.357
- - type: mrr_at_500
- value: 87.358
- - type: mrr_at_700
- value: 87.358
+ value: 90.29734129342121
- type: mrr_at_1000
- value: 87.358
+ value: 90.30017606259217
+ - type: mrr_at_20
+ value: 90.27225310310567
+ - type: mrr_at_3
+ value: 89.67364393427842
+ - type: mrr_at_5
+ value: 90.02408282691847
+ - type: nauc_map_at_1000_diff1
+ value: 13.996962490498193
+ - type: nauc_map_at_1000_max
+ value: 31.088341560915445
+ - type: nauc_map_at_100_diff1
+ value: 13.970448193188556
+ - type: nauc_map_at_100_max
+ value: 31.087684952220247
+ - type: nauc_map_at_10_diff1
+ value: 13.56293132566905
+ - type: nauc_map_at_10_max
+ value: 30.962644436915344
+ - type: nauc_map_at_1_diff1
+ value: 65.05715415545146
+ - type: nauc_map_at_1_max
+ value: 46.76841849306898
+ - type: nauc_map_at_20_diff1
+ value: 13.818721623046128
+ - type: nauc_map_at_20_max
+ value: 31.10628421308357
+ - type: nauc_map_at_3_diff1
+ value: 12.756796413891127
+ - type: nauc_map_at_3_max
+ value: 28.590858280202685
+ - type: nauc_map_at_5_diff1
+ value: 13.414706505922059
+ - type: nauc_map_at_5_max
+ value: 30.307305897135052
+ - type: nauc_mrr_at_1000_diff1
+ value: 65.25765367556968
+ - type: nauc_mrr_at_1000_max
+ value: 51.388832597229914
+ - type: nauc_mrr_at_100_diff1
+ value: 65.25925707496606
+ - type: nauc_mrr_at_100_max
+ value: 51.39863907803214
+ - type: nauc_mrr_at_10_diff1
+ value: 65.29285763562329
+ - type: nauc_mrr_at_10_max
+ value: 51.60050184164565
+ - type: nauc_mrr_at_1_diff1
+ value: 65.05715415545146
+ - type: nauc_mrr_at_1_max
+ value: 46.76841849306898
+ - type: nauc_mrr_at_20_diff1
+ value: 65.26501299828944
+ - type: nauc_mrr_at_20_max
+ value: 51.47458877298815
+ - type: nauc_mrr_at_3_diff1
+ value: 65.31017443516791
+ - type: nauc_mrr_at_3_max
+ value: 51.888681778858846
+ - type: nauc_mrr_at_5_diff1
+ value: 65.26016566846297
+ - type: nauc_mrr_at_5_max
+ value: 51.71174461060035
+ - type: nauc_ndcg_at_1000_diff1
+ value: 21.29033070562454
+ - type: nauc_ndcg_at_1000_max
+ value: 35.79875709229432
+ - type: nauc_ndcg_at_100_diff1
+ value: 20.508240016508587
+ - type: nauc_ndcg_at_100_max
+ value: 35.803726680555776
+ - type: nauc_ndcg_at_10_diff1
+ value: 18.733519691215687
+ - type: nauc_ndcg_at_10_max
+ value: 35.41479124285751
+ - type: nauc_ndcg_at_1_diff1
+ value: 65.05715415545146
+ - type: nauc_ndcg_at_1_max
+ value: 46.76841849306898
+ - type: nauc_ndcg_at_20_diff1
+ value: 19.4803450991088
+ - type: nauc_ndcg_at_20_max
+ value: 35.847270503972354
+ - type: nauc_ndcg_at_3_diff1
+ value: 17.662922721208943
+ - type: nauc_ndcg_at_3_max
+ value: 31.818069563062163
+ - type: nauc_ndcg_at_5_diff1
+ value: 18.398877245479625
+ - type: nauc_ndcg_at_5_max
+ value: 34.08295660833997
+ - type: nauc_precision_at_1000_diff1
+ value: 7.258155183986985
+ - type: nauc_precision_at_1000_max
+ value: 49.982656147243596
+ - type: nauc_precision_at_100_diff1
+ value: 4.250356736084608
+ - type: nauc_precision_at_100_max
+ value: 39.733886222600326
+ - type: nauc_precision_at_10_diff1
+ value: 5.186152765363542
+ - type: nauc_precision_at_10_max
+ value: 36.17781091027894
+ - type: nauc_precision_at_1_diff1
+ value: 65.05715415545146
+ - type: nauc_precision_at_1_max
+ value: 46.76841849306898
+ - type: nauc_precision_at_20_diff1
+ value: 4.804962592200048
+ - type: nauc_precision_at_20_max
+ value: 37.751940318199566
+ - type: nauc_precision_at_3_diff1
+ value: 7.606359228601881
+ - type: nauc_precision_at_3_max
+ value: 29.20046053844811
+ - type: nauc_precision_at_5_diff1
+ value: 6.8996932381953755
+ - type: nauc_precision_at_5_max
+ value: 32.76172656647554
+ - type: nauc_recall_at_1000_diff1
+ value: 7.258155183988313
+ - type: nauc_recall_at_1000_max
+ value: 49.982656147242686
+ - type: nauc_recall_at_100_diff1
+ value: 4.250356736084467
+ - type: nauc_recall_at_100_max
+ value: 39.733886222600454
+ - type: nauc_recall_at_10_diff1
+ value: 5.186152765363712
+ - type: nauc_recall_at_10_max
+ value: 36.17781091027895
+ - type: nauc_recall_at_1_diff1
+ value: 65.05715415545146
+ - type: nauc_recall_at_1_max
+ value: 46.76841849306898
+ - type: nauc_recall_at_20_diff1
+ value: 4.804962592199889
+ - type: nauc_recall_at_20_max
+ value: 37.75194031819925
+ - type: nauc_recall_at_3_diff1
+ value: 7.6063592286019155
+ - type: nauc_recall_at_3_max
+ value: 29.20046053844811
+ - type: nauc_recall_at_5_diff1
+ value: 6.899693238195505
+ - type: nauc_recall_at_5_max
+ value: 32.76172656647559
+ - type: ndcg_at_1
+ value: 85.483
+ - type: ndcg_at_10
+ value: 81.361
+ - type: ndcg_at_100
+ value: 83.588
+ - type: ndcg_at_1000
+ value: 84.19
+ - type: ndcg_at_20
+ value: 82.42699999999999
+ - type: ndcg_at_3
+ value: 76.779
+ - type: ndcg_at_5
+ value: 79.581
+ - type: precision_at_1
+ value: 85.483
+ - type: precision_at_10
+ value: 17.113
+ - type: precision_at_100
+ value: 1.882
+ - type: precision_at_1000
+ value: 0.196
+ - type: precision_at_20
+ value: 8.899
+ - type: precision_at_3
+ value: 50.397999999999996
+ - type: precision_at_5
+ value: 32.443
+ - type: recall_at_1
+ value: 42.741
+ - type: recall_at_10
+ value: 85.564
+ - type: recall_at_100
+ value: 94.07799999999999
+ - type: recall_at_1000
+ value: 97.995
+ - type: recall_at_20
+ value: 88.98700000000001
+ - type: recall_at_3
+ value: 75.598
+ - type: recall_at_5
+ value: 81.107
- task:
type: Classification
dataset:
@@ -1346,232 +1702,282 @@ model-index:
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7
metrics:
- type: accuracy
- value: 94.79200000000002
+ value: 96.80320000000002
- type: ap
- value: 92.54484356773553
+ value: 94.98856145360044
- type: f1
- value: 94.78965313682525
+ value: 96.80287885839178
- task:
type: Retrieval
dataset:
- type: msmarco
+ type: mteb/msmarco
name: MTEB MSMARCO
config: default
split: dev
- revision: None
+ revision: c5a29a104738b98a9e76336939199e264163d4a0
metrics:
- - type: ndcg_at_1
- value: 24.398
- - type: ndcg_at_2
- value: 31.336000000000002
- - type: ndcg_at_3
- value: 35.266999999999996
- - type: ndcg_at_5
- value: 39.356
- - type: ndcg_at_7
- value: 41.562
- - type: ndcg_at_10
- value: 43.408
- - type: ndcg_at_20
- value: 46.107
- - type: ndcg_at_30
- value: 47.164
- - type: ndcg_at_50
- value: 48.126000000000005
- - type: ndcg_at_70
- value: 48.626999999999995
- - type: ndcg_at_100
- value: 49.043
- - type: ndcg_at_200
- value: 49.575
- - type: ndcg_at_300
- value: 49.794
- - type: ndcg_at_500
- value: 49.942
- - type: ndcg_at_700
- value: 50.014
- - type: ndcg_at_1000
- value: 50.077000000000005
- type: map_at_1
- value: 23.723
- - type: map_at_2
- value: 29.593000000000004
- - type: map_at_3
- value: 32.273
- - type: map_at_5
- value: 34.587
- - type: map_at_7
- value: 35.589999999999996
+ value: 22.539
- type: map_at_10
- value: 36.296
- - type: map_at_20
- value: 37.059999999999995
- - type: map_at_30
- value: 37.265
- - type: map_at_50
- value: 37.402
- - type: map_at_70
- value: 37.454
+ value: 35.109
- type: map_at_100
- value: 37.486999999999995
- - type: map_at_200
- value: 37.516
- - type: map_at_300
- value: 37.524
- - type: map_at_500
- value: 37.528
- - type: map_at_700
- value: 37.529
+ value: 36.287000000000006
- type: map_at_1000
- value: 37.53
- - type: recall_at_1
- value: 23.723
- - type: recall_at_2
- value: 35.355
- - type: recall_at_3
- value: 43.22
- - type: recall_at_5
- value: 53.025
- - type: recall_at_7
- value: 59.327
- - type: recall_at_10
- value: 65.302
- - type: recall_at_20
- value: 75.765
- - type: recall_at_30
- value: 80.632
- - type: recall_at_50
- value: 85.63499999999999
- - type: recall_at_70
- value: 88.554
- - type: recall_at_100
- value: 91.16300000000001
- - type: recall_at_200
- value: 94.85
- - type: recall_at_300
- value: 96.532
- - type: recall_at_500
- value: 97.751
- - type: recall_at_700
- value: 98.383
- - type: recall_at_1000
- value: 98.97
+ value: 36.335
+ - type: map_at_20
+ value: 35.838
+ - type: map_at_3
+ value: 31.11
+ - type: map_at_5
+ value: 33.455
+ - type: mrr_at_1
+ value: 23.15186246418338
+ - type: mrr_at_10
+ value: 35.70532018920268
+ - type: mrr_at_100
+ value: 36.815167506137584
+ - type: mrr_at_1000
+ value: 36.85695349443505
+ - type: mrr_at_20
+ value: 36.39500867880642
+ - type: mrr_at_3
+ value: 31.81232091690535
+ - type: mrr_at_5
+ value: 34.096704871060155
+ - type: nauc_map_at_1000_diff1
+ value: 37.46807558257006
+ - type: nauc_map_at_1000_max
+ value: 4.458951631713913
+ - type: nauc_map_at_100_diff1
+ value: 37.46074371522825
+ - type: nauc_map_at_100_max
+ value: 4.468613496084279
+ - type: nauc_map_at_10_diff1
+ value: 37.26010875380019
+ - type: nauc_map_at_10_max
+ value: 4.231875758608358
+ - type: nauc_map_at_1_diff1
+ value: 40.65880154457905
+ - type: nauc_map_at_1_max
+ value: 4.630177805770112
+ - type: nauc_map_at_20_diff1
+ value: 37.38034955546114
+ - type: nauc_map_at_20_max
+ value: 4.387531851079199
+ - type: nauc_map_at_3_diff1
+ value: 37.29450703108972
+ - type: nauc_map_at_3_max
+ value: 4.041085172940792
+ - type: nauc_map_at_5_diff1
+ value: 37.193875198938684
+ - type: nauc_map_at_5_max
+ value: 4.255380917897142
+ - type: nauc_mrr_at_1000_diff1
+ value: 37.51703832662947
+ - type: nauc_mrr_at_1000_max
+ value: 4.700969406611156
+ - type: nauc_mrr_at_100_diff1
+ value: 37.509900052161484
+ - type: nauc_mrr_at_100_max
+ value: 4.711510132171848
+ - type: nauc_mrr_at_10_diff1
+ value: 37.318504251253096
+ - type: nauc_mrr_at_10_max
+ value: 4.509976853509513
+ - type: nauc_mrr_at_1_diff1
+ value: 40.87183740427009
+ - type: nauc_mrr_at_1_max
+ value: 4.9946480154946
+ - type: nauc_mrr_at_20_diff1
+ value: 37.44424499639884
+ - type: nauc_mrr_at_20_max
+ value: 4.6610703394064545
+ - type: nauc_mrr_at_3_diff1
+ value: 37.41339289659933
+ - type: nauc_mrr_at_3_max
+ value: 4.3843049357899355
+ - type: nauc_mrr_at_5_diff1
+ value: 37.2818206254161
+ - type: nauc_mrr_at_5_max
+ value: 4.515408685425186
+ - type: nauc_ndcg_at_1000_diff1
+ value: 37.10701525719374
+ - type: nauc_ndcg_at_1000_max
+ value: 5.094963991103502
+ - type: nauc_ndcg_at_100_diff1
+ value: 36.92131913545162
+ - type: nauc_ndcg_at_100_max
+ value: 5.315886593740655
+ - type: nauc_ndcg_at_10_diff1
+ value: 36.17872748228284
+ - type: nauc_ndcg_at_10_max
+ value: 4.277703493208093
+ - type: nauc_ndcg_at_1_diff1
+ value: 40.87183740427009
+ - type: nauc_ndcg_at_1_max
+ value: 4.9946480154946
+ - type: nauc_ndcg_at_20_diff1
+ value: 36.57195642236175
+ - type: nauc_ndcg_at_20_max
+ value: 4.811660629869289
+ - type: nauc_ndcg_at_3_diff1
+ value: 36.379621410103454
+ - type: nauc_ndcg_at_3_max
+ value: 3.9950698669171363
+ - type: nauc_ndcg_at_5_diff1
+ value: 36.12466303162312
+ - type: nauc_ndcg_at_5_max
+ value: 4.307405514081432
+ - type: nauc_precision_at_1000_diff1
+ value: 0.735503172068813
+ - type: nauc_precision_at_1000_max
+ value: 12.45145740028363
+ - type: nauc_precision_at_100_diff1
+ value: 18.100083143162525
+ - type: nauc_precision_at_100_max
+ value: 13.696889915097154
+ - type: nauc_precision_at_10_diff1
+ value: 30.466688930583942
+ - type: nauc_precision_at_10_max
+ value: 4.770700965741308
+ - type: nauc_precision_at_1_diff1
+ value: 40.87183740427009
+ - type: nauc_precision_at_1_max
+ value: 4.9946480154946
+ - type: nauc_precision_at_20_diff1
+ value: 28.754935817042416
+ - type: nauc_precision_at_20_max
+ value: 7.46499606719654
+ - type: nauc_precision_at_3_diff1
+ value: 33.678515329571226
+ - type: nauc_precision_at_3_max
+ value: 3.7689256210367144
+ - type: nauc_precision_at_5_diff1
+ value: 32.0270031155718
+ - type: nauc_precision_at_5_max
+ value: 4.744266280225248
+ - type: nauc_recall_at_1000_diff1
+ value: 42.87375080751249
+ - type: nauc_recall_at_1000_max
+ value: 44.65877854409723
+ - type: nauc_recall_at_100_diff1
+ value: 33.12747919897128
+ - type: nauc_recall_at_100_max
+ value: 14.932690625875052
+ - type: nauc_recall_at_10_diff1
+ value: 32.203513747598116
+ - type: nauc_recall_at_10_max
+ value: 4.083274933046108
+ - type: nauc_recall_at_1_diff1
+ value: 40.65880154457905
+ - type: nauc_recall_at_1_max
+ value: 4.630177805770112
+ - type: nauc_recall_at_20_diff1
+ value: 33.183598565798704
+ - type: nauc_recall_at_20_max
+ value: 6.474795407508596
+ - type: nauc_recall_at_3_diff1
+ value: 33.55391874721875
+ - type: nauc_recall_at_3_max
+ value: 3.5896556083093882
+ - type: nauc_recall_at_5_diff1
+ value: 32.69968929029696
+ - type: nauc_recall_at_5_max
+ value: 4.198811908890888
+ - type: ndcg_at_1
+ value: 23.152
+ - type: ndcg_at_10
+ value: 42.181999999999995
+ - type: ndcg_at_100
+ value: 47.847
+ - type: ndcg_at_1000
+ value: 48.988
+ - type: ndcg_at_20
+ value: 44.767
+ - type: ndcg_at_3
+ value: 34.088
+ - type: ndcg_at_5
+ value: 38.257999999999996
- type: precision_at_1
- value: 24.398
- - type: precision_at_2
- value: 18.274
- - type: precision_at_3
- value: 14.951999999999998
- - type: precision_at_5
- value: 11.052
- - type: precision_at_7
- value: 8.84
+ value: 23.152
- type: precision_at_10
- value: 6.8309999999999995
- - type: precision_at_20
- value: 3.978
- - type: precision_at_30
- value: 2.827
- - type: precision_at_50
- value: 1.807
- - type: precision_at_70
- value: 1.336
+ value: 6.678000000000001
- type: precision_at_100
- value: 0.964
- - type: precision_at_200
- value: 0.502
- - type: precision_at_300
- value: 0.34099999999999997
- - type: precision_at_500
- value: 0.208
- - type: precision_at_700
- value: 0.15
+ value: 0.9530000000000001
- type: precision_at_1000
value: 0.105
- - type: mrr_at_1
- value: 24.398
- - type: mrr_at_2
- value: 30.351
- - type: mrr_at_3
- value: 33.001000000000005
- - type: mrr_at_5
- value: 35.228
- - type: mrr_at_7
- value: 36.223
- - type: mrr_at_10
- value: 36.903999999999996
- - type: mrr_at_20
- value: 37.631
- - type: mrr_at_30
- value: 37.830000000000005
- - type: mrr_at_50
- value: 37.955
- - type: mrr_at_70
- value: 38.003
- - type: mrr_at_100
- value: 38.033
- - type: mrr_at_200
- value: 38.059
- - type: mrr_at_300
- value: 38.066
- - type: mrr_at_500
- value: 38.068999999999996
- - type: mrr_at_700
- value: 38.07
- - type: mrr_at_1000
- value: 38.07
+ - type: precision_at_20
+ value: 3.881
+ - type: precision_at_3
+ value: 14.518
+ - type: precision_at_5
+ value: 10.831
+ - type: recall_at_1
+ value: 22.539
+ - type: recall_at_10
+ value: 63.965
+ - type: recall_at_100
+ value: 90.129
+ - type: recall_at_1000
+ value: 98.721
+ - type: recall_at_20
+ value: 74.00999999999999
+ - type: recall_at_3
+ value: 42.004999999999995
+ - type: recall_at_5
+ value: 52.028
- task:
type: Classification
dataset:
type: mteb/mtop_domain
- name: MTEB MTOPDomainClassification (en)
- config: en
+ name: MTEB MTOPDomainClassification
+ config: default
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics:
- type: accuracy
- value: 96.35658914728683
+ value: 98.5750113999088
- type: f1
- value: 96.15039630903114
+ value: 98.41576079230245
- task:
type: Classification
dataset:
type: mteb/mtop_intent
- name: MTEB MTOPIntentClassification (en)
- config: en
+ name: MTEB MTOPIntentClassification
+ config: default
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics:
- type: accuracy
- value: 86.29730962152303
+ value: 91.29502963976289
- type: f1
- value: 71.12166316567485
+ value: 74.84400169335184
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
- name: MTEB MassiveIntentClassification (en)
- config: en
+ name: MTEB MassiveIntentClassification
+ config: default
split: test
- revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
+ revision: 4672e20407010da34463acc759c162ca9734bca6
metrics:
- type: accuracy
- value: 79.98991257565568
+ value: 85.96839273705447
- type: f1
- value: 77.41680115095276
+ value: 82.43129186593926
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
- name: MTEB MassiveScenarioClassification (en)
- config: en
+ name: MTEB MassiveScenarioClassification
+ config: default
split: test
- revision: 7d571f92784cd94a019292a1f45445077d0ef634
+ revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
metrics:
- type: accuracy
- value: 82.1990585070612
+ value: 90.60860793544047
- type: f1
- value: 82.23719179179362
+ value: 89.79415994859477
- task:
type: Clustering
dataset:
@@ -1582,7 +1988,7 @@ model-index:
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73
metrics:
- type: v_measure
- value: 40.03019554933584
+ value: 46.661892807041355
- task:
type: Clustering
dataset:
@@ -1593,7 +1999,7 @@ model-index:
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663
metrics:
- type: v_measure
- value: 38.999760551497815
+ value: 44.17598473858937
- task:
type: Reranking
dataset:
@@ -1601,519 +2007,677 @@ model-index:
name: MTEB MindSmallReranking
config: default
split: test
- revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69
+ revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7
metrics:
- type: map
- value: 32.72383151953079
+ value: 31.260919294024603
- type: mrr
- value: 33.93989699030721
+ value: 32.37049108835034
+ - type: nAUC_map_diff1
+ value: 9.41443010763114
+ - type: nAUC_map_max
+ value: -21.43467269970586
+ - type: nAUC_mrr_diff1
+ value: 9.296395912186979
+ - type: nAUC_mrr_max
+ value: -15.969133314683132
- task:
type: Retrieval
dataset:
- type: nfcorpus
+ type: mteb/nfcorpus
name: MTEB NFCorpus
config: default
split: test
- revision: None
+ revision: ec0fa4fe99da2ff19ca1214b7966684033a58814
metrics:
- - type: ndcg_at_1
- value: 51.858000000000004
- - type: ndcg_at_2
- value: 49.675999999999995
- - type: ndcg_at_3
- value: 47.519
- - type: ndcg_at_5
- value: 45.198
- - type: ndcg_at_7
- value: 43.504
- - type: ndcg_at_10
- value: 41.88
- - type: ndcg_at_20
- value: 39.122
- - type: ndcg_at_30
- value: 37.95
- - type: ndcg_at_50
- value: 37.602999999999994
- - type: ndcg_at_70
- value: 37.836
- - type: ndcg_at_100
- value: 38.493
- - type: ndcg_at_200
- value: 40.187
- - type: ndcg_at_300
- value: 41.524
- - type: ndcg_at_500
- value: 43.657000000000004
- - type: ndcg_at_700
- value: 45.234
- - type: ndcg_at_1000
- value: 47.047
- type: map_at_1
- value: 6.392
- - type: map_at_2
- value: 10.113
- - type: map_at_3
- value: 11.543000000000001
- - type: map_at_5
- value: 13.729
- - type: map_at_7
- value: 14.985000000000001
+ value: 6.672000000000001
- type: map_at_10
- value: 16.217000000000002
- - type: map_at_20
- value: 18.106
- - type: map_at_30
- value: 18.878
- - type: map_at_50
- value: 19.822
- - type: map_at_70
- value: 20.352999999999998
+ value: 15.972
- type: map_at_100
- value: 20.827
- - type: map_at_200
- value: 21.512
- - type: map_at_300
- value: 21.826
- - type: map_at_500
- value: 22.155
- - type: map_at_700
- value: 22.349
+ value: 20.94
- type: map_at_1000
- value: 22.531000000000002
- - type: recall_at_1
- value: 6.392
- - type: recall_at_2
- value: 11.215
- - type: recall_at_3
- value: 13.231000000000002
- - type: recall_at_5
- value: 16.66
- - type: recall_at_7
- value: 18.802
- - type: recall_at_10
- value: 21.185000000000002
- - type: recall_at_20
- value: 25.35
- - type: recall_at_30
- value: 27.91
- - type: recall_at_50
- value: 32.845
- - type: recall_at_70
- value: 35.789
- - type: recall_at_100
- value: 39.247
- - type: recall_at_200
- value: 46.655
- - type: recall_at_300
- value: 51.43299999999999
- - type: recall_at_500
- value: 59.472
- - type: recall_at_700
- value: 64.742
- - type: recall_at_1000
- value: 70.97099999999999
- - type: precision_at_1
- value: 53.559999999999995
- - type: precision_at_2
- value: 48.762
- - type: precision_at_3
- value: 44.169000000000004
- - type: precision_at_5
- value: 39.071
- - type: precision_at_7
- value: 35.161
- - type: precision_at_10
- value: 31.238
- - type: precision_at_20
- value: 23.064999999999998
- - type: precision_at_30
- value: 18.844
- - type: precision_at_50
- value: 14.601
- - type: precision_at_70
- value: 12.088000000000001
- - type: precision_at_100
- value: 9.844999999999999
- - type: precision_at_200
- value: 6.358
- - type: precision_at_300
- value: 4.915
- - type: precision_at_500
- value: 3.531
- - type: precision_at_700
- value: 2.8649999999999998
- - type: precision_at_1000
- value: 2.289
+ value: 22.877
+ - type: map_at_20
+ value: 17.986
+ - type: map_at_3
+ value: 11.161
+ - type: map_at_5
+ value: 13.293
- type: mrr_at_1
- value: 54.17999999999999
- - type: mrr_at_2
- value: 59.288
- - type: mrr_at_3
- value: 60.836
- - type: mrr_at_5
- value: 62.275999999999996
- - type: mrr_at_7
- value: 62.688
+ value: 53.56037151702786
- type: mrr_at_10
- value: 62.865
- - type: mrr_at_20
- value: 63.11
- - type: mrr_at_30
- value: 63.193999999999996
- - type: mrr_at_50
- value: 63.258
- - type: mrr_at_70
- value: 63.278
+ value: 61.915696103002595
- type: mrr_at_100
- value: 63.297000000000004
- - type: mrr_at_200
- value: 63.315999999999995
- - type: mrr_at_300
- value: 63.318
- - type: mrr_at_500
- value: 63.32299999999999
- - type: mrr_at_700
- value: 63.324000000000005
+ value: 62.4130902631107
- type: mrr_at_1000
- value: 63.324999999999996
+ value: 62.45228087711845
+ - type: mrr_at_20
+ value: 62.1983715004112
+ - type: mrr_at_3
+ value: 60.31991744066049
+ - type: mrr_at_5
+ value: 61.27966976264191
+ - type: nauc_map_at_1000_diff1
+ value: 19.760261389791232
+ - type: nauc_map_at_1000_max
+ value: 44.92884959603954
+ - type: nauc_map_at_100_diff1
+ value: 21.434054251654597
+ - type: nauc_map_at_100_max
+ value: 44.411868533880984
+ - type: nauc_map_at_10_diff1
+ value: 26.466765779031043
+ - type: nauc_map_at_10_max
+ value: 38.064663610865296
+ - type: nauc_map_at_1_diff1
+ value: 44.574185416582196
+ - type: nauc_map_at_1_max
+ value: 21.04217118273998
+ - type: nauc_map_at_20_diff1
+ value: 24.17451575612751
+ - type: nauc_map_at_20_max
+ value: 41.57393313672135
+ - type: nauc_map_at_3_diff1
+ value: 35.15830484508489
+ - type: nauc_map_at_3_max
+ value: 26.877457835109286
+ - type: nauc_map_at_5_diff1
+ value: 30.83258164353464
+ - type: nauc_map_at_5_max
+ value: 33.19797207731497
+ - type: nauc_mrr_at_1000_diff1
+ value: 31.1416556460293
+ - type: nauc_mrr_at_1000_max
+ value: 58.87534131082168
+ - type: nauc_mrr_at_100_diff1
+ value: 31.15241841441478
+ - type: nauc_mrr_at_100_max
+ value: 58.88635397501152
+ - type: nauc_mrr_at_10_diff1
+ value: 31.260718973796592
+ - type: nauc_mrr_at_10_max
+ value: 58.806194826266776
+ - type: nauc_mrr_at_1_diff1
+ value: 33.760222861133585
+ - type: nauc_mrr_at_1_max
+ value: 54.96698629743225
+ - type: nauc_mrr_at_20_diff1
+ value: 31.12813913710054
+ - type: nauc_mrr_at_20_max
+ value: 58.79610076856905
+ - type: nauc_mrr_at_3_diff1
+ value: 31.701540852177107
+ - type: nauc_mrr_at_3_max
+ value: 58.433134491512405
+ - type: nauc_mrr_at_5_diff1
+ value: 30.969066655800585
+ - type: nauc_mrr_at_5_max
+ value: 59.03858582179269
+ - type: nauc_ndcg_at_1000_diff1
+ value: 17.06336834873998
+ - type: nauc_ndcg_at_1000_max
+ value: 55.381237569833594
+ - type: nauc_ndcg_at_100_diff1
+ value: 17.14638339092883
+ - type: nauc_ndcg_at_100_max
+ value: 50.74323374985141
+ - type: nauc_ndcg_at_10_diff1
+ value: 15.049271508549172
+ - type: nauc_ndcg_at_10_max
+ value: 51.757712306479256
+ - type: nauc_ndcg_at_1_diff1
+ value: 33.41794124902274
+ - type: nauc_ndcg_at_1_max
+ value: 49.721776929969
+ - type: nauc_ndcg_at_20_diff1
+ value: 14.657837541049549
+ - type: nauc_ndcg_at_20_max
+ value: 47.70520745742888
+ - type: nauc_ndcg_at_3_diff1
+ value: 22.730121098035067
+ - type: nauc_ndcg_at_3_max
+ value: 51.69547926227459
+ - type: nauc_ndcg_at_5_diff1
+ value: 19.693727104598903
+ - type: nauc_ndcg_at_5_max
+ value: 53.98564853608311
+ - type: nauc_precision_at_1000_diff1
+ value: -17.707024623502395
+ - type: nauc_precision_at_1000_max
+ value: 4.211934757014257
+ - type: nauc_precision_at_100_diff1
+ value: -15.149938798161827
+ - type: nauc_precision_at_100_max
+ value: 19.303331670820295
+ - type: nauc_precision_at_10_diff1
+ value: -5.749815345240198
+ - type: nauc_precision_at_10_max
+ value: 44.32432083215351
+ - type: nauc_precision_at_1_diff1
+ value: 34.563752660017315
+ - type: nauc_precision_at_1_max
+ value: 53.19426094095173
+ - type: nauc_precision_at_20_diff1
+ value: -9.119453895462609
+ - type: nauc_precision_at_20_max
+ value: 34.85164292348402
+ - type: nauc_precision_at_3_diff1
+ value: 11.112561909548212
+ - type: nauc_precision_at_3_max
+ value: 51.08405979799506
+ - type: nauc_precision_at_5_diff1
+ value: 3.3910084651331065
+ - type: nauc_precision_at_5_max
+ value: 50.855893129778075
+ - type: nauc_recall_at_1000_diff1
+ value: -1.58295745759997
+ - type: nauc_recall_at_1000_max
+ value: 24.3670553238421
+ - type: nauc_recall_at_100_diff1
+ value: 11.118620671336878
+ - type: nauc_recall_at_100_max
+ value: 31.364616111695348
+ - type: nauc_recall_at_10_diff1
+ value: 17.866983689612564
+ - type: nauc_recall_at_10_max
+ value: 29.497193026002833
+ - type: nauc_recall_at_1_diff1
+ value: 44.574185416582196
+ - type: nauc_recall_at_1_max
+ value: 21.04217118273998
+ - type: nauc_recall_at_20_diff1
+ value: 14.497586041161275
+ - type: nauc_recall_at_20_max
+ value: 29.715986816551837
+ - type: nauc_recall_at_3_diff1
+ value: 31.19718292095584
+ - type: nauc_recall_at_3_max
+ value: 24.254608039842722
+ - type: nauc_recall_at_5_diff1
+ value: 25.49206341904369
+ - type: nauc_recall_at_5_max
+ value: 30.7070758478257
+ - type: ndcg_at_1
+ value: 50.929
+ - type: ndcg_at_10
+ value: 41.336
+ - type: ndcg_at_100
+ value: 38.586999999999996
+ - type: ndcg_at_1000
+ value: 48.155
+ - type: ndcg_at_20
+ value: 38.888
+ - type: ndcg_at_3
+ value: 47.0
+ - type: ndcg_at_5
+ value: 44.335
+ - type: precision_at_1
+ value: 53.251000000000005
+ - type: precision_at_10
+ value: 31.146
+ - type: precision_at_100
+ value: 10.040000000000001
+ - type: precision_at_1000
+ value: 2.432
+ - type: precision_at_20
+ value: 23.421
+ - type: precision_at_3
+ value: 45.098
+ - type: precision_at_5
+ value: 39.071
+ - type: recall_at_1
+ value: 6.672000000000001
+ - type: recall_at_10
+ value: 20.764
+ - type: recall_at_100
+ value: 40.759
+ - type: recall_at_1000
+ value: 75.015
+ - type: recall_at_20
+ value: 25.548
+ - type: recall_at_3
+ value: 12.328
+ - type: recall_at_5
+ value: 15.601999999999999
- task:
type: Retrieval
dataset:
- type: nq
+ type: mteb/nq
name: MTEB NQ
config: default
split: test
- revision: None
+ revision: b774495ed302d8c44a3a7ea25c90dbce03968f31
metrics:
- - type: ndcg_at_1
- value: 50.897999999999996
- - type: ndcg_at_2
- value: 59.126
- - type: ndcg_at_3
- value: 63.093999999999994
- - type: ndcg_at_5
- value: 67.197
- - type: ndcg_at_7
- value: 68.719
- - type: ndcg_at_10
- value: 69.915
- - type: ndcg_at_20
- value: 71.229
- - type: ndcg_at_30
- value: 71.667
- - type: ndcg_at_50
- value: 71.98
- - type: ndcg_at_70
- value: 72.127
- - type: ndcg_at_100
- value: 72.217
- - type: ndcg_at_200
- value: 72.319
- - type: ndcg_at_300
- value: 72.347
- - type: ndcg_at_500
- value: 72.37
- - type: ndcg_at_700
- value: 72.379
- - type: ndcg_at_1000
- value: 72.381
- type: map_at_1
- value: 45.297
- - type: map_at_2
- value: 55.596000000000004
- - type: map_at_3
- value: 58.724
- - type: map_at_5
- value: 61.387
- - type: map_at_7
- value: 62.173
+ value: 50.944
- type: map_at_10
- value: 62.69
- - type: map_at_20
- value: 63.125
- - type: map_at_30
- value: 63.223
- - type: map_at_50
- value: 63.27700000000001
- - type: map_at_70
- value: 63.295
+ value: 67.565
- type: map_at_100
- value: 63.303
- - type: map_at_200
- value: 63.31
- - type: map_at_300
- value: 63.31099999999999
- - type: map_at_500
- value: 63.312000000000005
- - type: map_at_700
- value: 63.312000000000005
+ value: 68.10300000000001
- type: map_at_1000
- value: 63.312000000000005
- - type: recall_at_1
- value: 45.297
- - type: recall_at_2
- value: 63.866
- - type: recall_at_3
- value: 71.898
- - type: recall_at_5
- value: 81.16600000000001
- - type: recall_at_7
- value: 85.301
- - type: recall_at_10
- value: 88.94800000000001
- - type: recall_at_20
- value: 93.719
- - type: recall_at_30
- value: 95.628
- - type: recall_at_50
- value: 97.14699999999999
- - type: recall_at_70
- value: 97.955
- - type: recall_at_100
- value: 98.48599999999999
- - type: recall_at_200
- value: 99.157
- - type: recall_at_300
- value: 99.355
- - type: recall_at_500
- value: 99.53699999999999
- - type: recall_at_700
- value: 99.62299999999999
- - type: recall_at_1000
- value: 99.638
+ value: 68.109
+ - type: map_at_20
+ value: 67.973
+ - type: map_at_3
+ value: 64.176
+ - type: map_at_5
+ value: 66.39699999999999
+ - type: mrr_at_1
+ value: 57.01042873696408
+ - type: mrr_at_10
+ value: 69.76629605105849
+ - type: mrr_at_100
+ value: 70.09927347130204
+ - type: mrr_at_1000
+ value: 70.10309675839956
+ - type: mrr_at_20
+ value: 70.02288627712392
+ - type: mrr_at_3
+ value: 67.46813441483191
+ - type: mrr_at_5
+ value: 68.93105446118189
+ - type: nauc_map_at_1000_diff1
+ value: 46.365239710957376
+ - type: nauc_map_at_1000_max
+ value: 35.45336305992461
+ - type: nauc_map_at_100_diff1
+ value: 46.36894174569876
+ - type: nauc_map_at_100_max
+ value: 35.462248259280855
+ - type: nauc_map_at_10_diff1
+ value: 46.342944879162914
+ - type: nauc_map_at_10_max
+ value: 35.7114655816454
+ - type: nauc_map_at_1_diff1
+ value: 48.872462637268285
+ - type: nauc_map_at_1_max
+ value: 27.773826650545935
+ - type: nauc_map_at_20_diff1
+ value: 46.322116301176294
+ - type: nauc_map_at_20_max
+ value: 35.48090418392867
+ - type: nauc_map_at_3_diff1
+ value: 45.8339171755868
+ - type: nauc_map_at_3_max
+ value: 34.781034732538416
+ - type: nauc_map_at_5_diff1
+ value: 45.86777767034287
+ - type: nauc_map_at_5_max
+ value: 35.72520128399501
+ - type: nauc_mrr_at_1000_diff1
+ value: 46.54267081030922
+ - type: nauc_mrr_at_1000_max
+ value: 36.377090112491516
+ - type: nauc_mrr_at_100_diff1
+ value: 46.545857737076865
+ - type: nauc_mrr_at_100_max
+ value: 36.384356597091255
+ - type: nauc_mrr_at_10_diff1
+ value: 46.4713107252758
+ - type: nauc_mrr_at_10_max
+ value: 36.63520323903643
+ - type: nauc_mrr_at_1_diff1
+ value: 48.88656307461428
+ - type: nauc_mrr_at_1_max
+ value: 32.299595920907116
+ - type: nauc_mrr_at_20_diff1
+ value: 46.49398335076446
+ - type: nauc_mrr_at_20_max
+ value: 36.43755453427578
+ - type: nauc_mrr_at_3_diff1
+ value: 46.10957375937073
+ - type: nauc_mrr_at_3_max
+ value: 36.50560739385579
+ - type: nauc_mrr_at_5_diff1
+ value: 46.01288332431176
+ - type: nauc_mrr_at_5_max
+ value: 36.848142511421514
+ - type: nauc_ndcg_at_1000_diff1
+ value: 46.18453938546787
+ - type: nauc_ndcg_at_1000_max
+ value: 36.85054798292308
+ - type: nauc_ndcg_at_100_diff1
+ value: 46.29662813740788
+ - type: nauc_ndcg_at_100_max
+ value: 37.13686252016967
+ - type: nauc_ndcg_at_10_diff1
+ value: 45.91731104038101
+ - type: nauc_ndcg_at_10_max
+ value: 38.273169754276296
+ - type: nauc_ndcg_at_1_diff1
+ value: 48.88656307461428
+ - type: nauc_ndcg_at_1_max
+ value: 32.299595920907116
+ - type: nauc_ndcg_at_20_diff1
+ value: 45.92148394918942
+ - type: nauc_ndcg_at_20_max
+ value: 37.46416170260353
+ - type: nauc_ndcg_at_3_diff1
+ value: 44.843506024611344
+ - type: nauc_ndcg_at_3_max
+ value: 36.823857363469195
+ - type: nauc_ndcg_at_5_diff1
+ value: 44.69169359935397
+ - type: nauc_ndcg_at_5_max
+ value: 38.25617746583364
+ - type: nauc_precision_at_1000_diff1
+ value: -15.96421980695039
+ - type: nauc_precision_at_1000_max
+ value: 2.6026640263637417
+ - type: nauc_precision_at_100_diff1
+ value: -13.598597401656706
+ - type: nauc_precision_at_100_max
+ value: 5.509181986779547
+ - type: nauc_precision_at_10_diff1
+ value: -0.7561927117593065
+ - type: nauc_precision_at_10_max
+ value: 19.78176483956855
+ - type: nauc_precision_at_1_diff1
+ value: 48.88656307461428
+ - type: nauc_precision_at_1_max
+ value: 32.299595920907116
+ - type: nauc_precision_at_20_diff1
+ value: -8.313981469275115
+ - type: nauc_precision_at_20_max
+ value: 11.99308431594538
+ - type: nauc_precision_at_3_diff1
+ value: 16.771450780145464
+ - type: nauc_precision_at_3_max
+ value: 30.9942447136273
+ - type: nauc_precision_at_5_diff1
+ value: 6.621821328313667
+ - type: nauc_precision_at_5_max
+ value: 26.412248375662813
+ - type: nauc_recall_at_1000_diff1
+ value: 55.12969601161647
+ - type: nauc_recall_at_1000_max
+ value: 87.41795313689475
+ - type: nauc_recall_at_100_diff1
+ value: 60.175682302057446
+ - type: nauc_recall_at_100_max
+ value: 80.11543190835057
+ - type: nauc_recall_at_10_diff1
+ value: 41.62419550096612
+ - type: nauc_recall_at_10_max
+ value: 52.87526291547873
+ - type: nauc_recall_at_1_diff1
+ value: 48.872462637268285
+ - type: nauc_recall_at_1_max
+ value: 27.773826650545935
+ - type: nauc_recall_at_20_diff1
+ value: 40.843898765289254
+ - type: nauc_recall_at_20_max
+ value: 55.34115103001523
+ - type: nauc_recall_at_3_diff1
+ value: 39.728470536029675
+ - type: nauc_recall_at_3_max
+ value: 40.79059216022094
+ - type: nauc_recall_at_5_diff1
+ value: 37.18324769281485
+ - type: nauc_recall_at_5_max
+ value: 46.62408466485096
+ - type: ndcg_at_1
+ value: 57.010000000000005
+ - type: ndcg_at_10
+ value: 73.956
+ - type: ndcg_at_100
+ value: 75.90299999999999
+ - type: ndcg_at_1000
+ value: 76.03999999999999
+ - type: ndcg_at_20
+ value: 75.17
+ - type: ndcg_at_3
+ value: 68.13900000000001
+ - type: ndcg_at_5
+ value: 71.532
- type: precision_at_1
- value: 50.897999999999996
- - type: precision_at_2
- value: 36.703
- - type: precision_at_3
- value: 27.926000000000002
- - type: precision_at_5
- value: 19.276
- - type: precision_at_7
- value: 14.533999999999999
+ value: 57.010000000000005
- type: precision_at_10
- value: 10.678
- - type: precision_at_20
- value: 5.663
- - type: precision_at_30
- value: 3.8600000000000003
- - type: precision_at_50
- value: 2.358
- - type: precision_at_70
- value: 1.7000000000000002
+ value: 10.91
- type: precision_at_100
- value: 1.198
- - type: precision_at_200
- value: 0.603
- - type: precision_at_300
- value: 0.40299999999999997
- - type: precision_at_500
- value: 0.242
- - type: precision_at_700
- value: 0.173
+ value: 1.2
- type: precision_at_1000
value: 0.121
- - type: mrr_at_1
- value: 50.897999999999996
- - type: mrr_at_2
- value: 59.994
- - type: mrr_at_3
- value: 62.553000000000004
- - type: mrr_at_5
- value: 64.307
- - type: mrr_at_7
- value: 64.864
- - type: mrr_at_10
- value: 65.22200000000001
- - type: mrr_at_20
- value: 65.499
- - type: mrr_at_30
- value: 65.561
- - type: mrr_at_50
- value: 65.592
- - type: mrr_at_70
- value: 65.602
- - type: mrr_at_100
- value: 65.607
- - type: mrr_at_200
- value: 65.61099999999999
- - type: mrr_at_300
- value: 65.61200000000001
- - type: mrr_at_500
- value: 65.61200000000001
- - type: mrr_at_700
- value: 65.61200000000001
- - type: mrr_at_1000
- value: 65.61200000000001
+ - type: precision_at_20
+ value: 5.753
+ - type: precision_at_3
+ value: 29.828
+ - type: precision_at_5
+ value: 19.971
+ - type: recall_at_1
+ value: 50.944
+ - type: recall_at_10
+ value: 90.754
+ - type: recall_at_100
+ value: 98.699
+ - type: recall_at_1000
+ value: 99.701
+ - type: recall_at_20
+ value: 95.148
+ - type: recall_at_3
+ value: 76.224
+ - type: recall_at_5
+ value: 83.872
- task:
type: Retrieval
dataset:
- type: quora
+ type: mteb/quora
name: MTEB QuoraRetrieval
config: default
split: test
- revision: None
+ revision: e4e08e0b7dbe3c8700f0daef558ff32256715259
metrics:
- - type: ndcg_at_1
- value: 82.96
- - type: ndcg_at_2
- value: 85.614
- - type: ndcg_at_3
- value: 87.19
- - type: ndcg_at_5
- value: 88.654
- - type: ndcg_at_7
- value: 89.287
- - type: ndcg_at_10
- value: 89.785
- - type: ndcg_at_20
- value: 90.384
- - type: ndcg_at_30
- value: 90.589
- - type: ndcg_at_50
- value: 90.738
- - type: ndcg_at_70
- value: 90.789
- - type: ndcg_at_100
- value: 90.824
- - type: ndcg_at_200
- value: 90.869
- - type: ndcg_at_300
- value: 90.881
- - type: ndcg_at_500
- value: 90.886
- - type: ndcg_at_700
- value: 90.889
- - type: ndcg_at_1000
- value: 90.889
- type: map_at_1
- value: 72.152
- - type: map_at_2
- value: 80.818
- - type: map_at_3
- value: 83.462
- - type: map_at_5
- value: 85.286
- - type: map_at_7
- value: 85.921
+ value: 71.856
- type: map_at_10
- value: 86.334
- - type: map_at_20
- value: 86.737
- - type: map_at_30
- value: 86.847
- - type: map_at_50
- value: 86.911
- - type: map_at_70
- value: 86.932
+ value: 86.077
- type: map_at_100
- value: 86.943
- - type: map_at_200
- value: 86.953
- - type: map_at_300
- value: 86.955
- - type: map_at_500
- value: 86.956
- - type: map_at_700
- value: 86.956
+ value: 86.696
- type: map_at_1000
- value: 86.956
- - type: recall_at_1
- value: 72.152
- - type: recall_at_2
- value: 84.129
- - type: recall_at_3
- value: 88.87
- - type: recall_at_5
- value: 93.067
- - type: recall_at_7
- value: 94.882
- - type: recall_at_10
- value: 96.353
- - type: recall_at_20
- value: 98.26700000000001
- - type: recall_at_30
- value: 98.92999999999999
- - type: recall_at_50
- value: 99.441
- - type: recall_at_70
- value: 99.619
- - type: recall_at_100
- value: 99.748
- - type: recall_at_200
- value: 99.911
- - type: recall_at_300
- value: 99.956
- - type: recall_at_500
- value: 99.98
- - type: recall_at_700
- value: 99.991
- - type: recall_at_1000
- value: 99.996
+ value: 86.708
+ - type: map_at_20
+ value: 86.493
+ - type: map_at_3
+ value: 83.176
+ - type: map_at_5
+ value: 85.008
+ - type: mrr_at_1
+ value: 82.74000000000001
+ - type: mrr_at_10
+ value: 88.68947222222207
+ - type: mrr_at_100
+ value: 88.78196949571182
+ - type: mrr_at_1000
+ value: 88.78223256200576
+ - type: mrr_at_20
+ value: 88.76455636228219
+ - type: mrr_at_3
+ value: 87.85833333333316
+ - type: mrr_at_5
+ value: 88.43933333333311
+ - type: nauc_map_at_1000_diff1
+ value: 77.83205196572432
+ - type: nauc_map_at_1000_max
+ value: 36.83053791329062
+ - type: nauc_map_at_100_diff1
+ value: 77.83967417990776
+ - type: nauc_map_at_100_max
+ value: 36.820189914925834
+ - type: nauc_map_at_10_diff1
+ value: 78.00602410945338
+ - type: nauc_map_at_10_max
+ value: 36.3801794207628
+ - type: nauc_map_at_1_diff1
+ value: 81.57410902403387
+ - type: nauc_map_at_1_max
+ value: 27.368526445751744
+ - type: nauc_map_at_20_diff1
+ value: 77.92185473277556
+ - type: nauc_map_at_20_max
+ value: 36.69802931988695
+ - type: nauc_map_at_3_diff1
+ value: 78.55106145188513
+ - type: nauc_map_at_3_max
+ value: 33.75078534684343
+ - type: nauc_map_at_5_diff1
+ value: 78.22568876897226
+ - type: nauc_map_at_5_max
+ value: 35.36466972515817
+ - type: nauc_mrr_at_1000_diff1
+ value: 78.47061099703544
+ - type: nauc_mrr_at_1000_max
+ value: 39.75757225464997
+ - type: nauc_mrr_at_100_diff1
+ value: 78.47089032120223
+ - type: nauc_mrr_at_100_max
+ value: 39.75841805411614
+ - type: nauc_mrr_at_10_diff1
+ value: 78.46374600192402
+ - type: nauc_mrr_at_10_max
+ value: 39.73366051026909
+ - type: nauc_mrr_at_1_diff1
+ value: 79.17927467190566
+ - type: nauc_mrr_at_1_max
+ value: 39.06758577736873
+ - type: nauc_mrr_at_20_diff1
+ value: 78.48101600370325
+ - type: nauc_mrr_at_20_max
+ value: 39.76499738151275
+ - type: nauc_mrr_at_3_diff1
+ value: 78.22679185815186
+ - type: nauc_mrr_at_3_max
+ value: 39.8541411158529
+ - type: nauc_mrr_at_5_diff1
+ value: 78.40268685228129
+ - type: nauc_mrr_at_5_max
+ value: 39.97056259854404
+ - type: nauc_ndcg_at_1000_diff1
+ value: 77.80824110316115
+ - type: nauc_ndcg_at_1000_max
+ value: 38.314791792086105
+ - type: nauc_ndcg_at_100_diff1
+ value: 77.85325339908903
+ - type: nauc_ndcg_at_100_max
+ value: 38.27309056760618
+ - type: nauc_ndcg_at_10_diff1
+ value: 77.84156646709303
+ - type: nauc_ndcg_at_10_max
+ value: 37.512055122939216
+ - type: nauc_ndcg_at_1_diff1
+ value: 79.17927467190566
+ - type: nauc_ndcg_at_1_max
+ value: 39.15903317064575
+ - type: nauc_ndcg_at_20_diff1
+ value: 77.98831716876829
+ - type: nauc_ndcg_at_20_max
+ value: 38.05810189777102
+ - type: nauc_ndcg_at_3_diff1
+ value: 77.22076859239452
+ - type: nauc_ndcg_at_3_max
+ value: 36.4423593890435
+ - type: nauc_ndcg_at_5_diff1
+ value: 77.60866885900089
+ - type: nauc_ndcg_at_5_max
+ value: 37.10893342249924
+ - type: nauc_precision_at_1000_diff1
+ value: -45.18569850675969
+ - type: nauc_precision_at_1000_max
+ value: -7.610620867113754
+ - type: nauc_precision_at_100_diff1
+ value: -44.87041671763768
+ - type: nauc_precision_at_100_max
+ value: -7.281553584548097
+ - type: nauc_precision_at_10_diff1
+ value: -40.45638430368999
+ - type: nauc_precision_at_10_max
+ value: -2.4431089932642003
+ - type: nauc_precision_at_1_diff1
+ value: 79.17927467190566
+ - type: nauc_precision_at_1_max
+ value: 39.15903317064575
+ - type: nauc_precision_at_20_diff1
+ value: -43.19241665329079
+ - type: nauc_precision_at_20_max
+ value: -4.867082362790704
+ - type: nauc_precision_at_3_diff1
+ value: -22.626190596449668
+ - type: nauc_precision_at_3_max
+ value: 6.979549396518983
+ - type: nauc_precision_at_5_diff1
+ value: -33.7901896548388
+ - type: nauc_precision_at_5_max
+ value: 1.6896971649623012
+ - type: nauc_recall_at_1000_diff1
+ value: 100.0
+ - type: nauc_recall_at_1000_max
+ value: 86.92810457506096
+ - type: nauc_recall_at_100_diff1
+ value: 80.69710836553755
+ - type: nauc_recall_at_100_max
+ value: 38.295085681104354
+ - type: nauc_recall_at_10_diff1
+ value: 75.08994082341206
+ - type: nauc_recall_at_10_max
+ value: 32.92423308069912
+ - type: nauc_recall_at_1_diff1
+ value: 81.57410902403387
+ - type: nauc_recall_at_1_max
+ value: 27.368526445751744
+ - type: nauc_recall_at_20_diff1
+ value: 77.59126784952974
+ - type: nauc_recall_at_20_max
+ value: 36.39562927549683
+ - type: nauc_recall_at_3_diff1
+ value: 74.7562435650145
+ - type: nauc_recall_at_3_max
+ value: 30.37160131415078
+ - type: nauc_recall_at_5_diff1
+ value: 74.00625102274935
+ - type: nauc_recall_at_5_max
+ value: 32.53548145587051
+ - type: ndcg_at_1
+ value: 82.74000000000001
+ - type: ndcg_at_10
+ value: 89.583
+ - type: ndcg_at_100
+ value: 90.652
+ - type: ndcg_at_1000
+ value: 90.711
+ - type: ndcg_at_20
+ value: 90.203
+ - type: ndcg_at_3
+ value: 86.967
+ - type: ndcg_at_5
+ value: 88.43299999999999
- type: precision_at_1
- value: 82.96
- - type: precision_at_2
- value: 52.175000000000004
- - type: precision_at_3
- value: 38.223
- - type: precision_at_5
- value: 25.056
- - type: precision_at_7
- value: 18.717
+ value: 82.74000000000001
- type: precision_at_10
- value: 13.614999999999998
- - type: precision_at_20
- value: 7.208
- - type: precision_at_30
- value: 4.928
- - type: precision_at_50
- value: 3.024
- - type: precision_at_70
- value: 2.183
+ value: 13.617
- type: precision_at_100
- value: 1.54
- - type: precision_at_200
- value: 0.779
- - type: precision_at_300
- value: 0.521
- - type: precision_at_500
- value: 0.313
- - type: precision_at_700
- value: 0.22399999999999998
+ value: 1.542
- type: precision_at_1000
value: 0.157
- - type: mrr_at_1
- value: 82.96
- - type: mrr_at_2
- value: 87.005
- - type: mrr_at_3
- value: 88.07199999999999
- - type: mrr_at_5
- value: 88.634
- - type: mrr_at_7
- value: 88.793
- - type: mrr_at_10
- value: 88.87899999999999
- - type: mrr_at_20
- value: 88.94999999999999
- - type: mrr_at_30
- value: 88.96
- - type: mrr_at_50
- value: 88.965
- - type: mrr_at_70
- value: 88.966
- - type: mrr_at_100
- value: 88.967
- - type: mrr_at_200
- value: 88.967
- - type: mrr_at_300
- value: 88.967
- - type: mrr_at_500
- value: 88.967
- - type: mrr_at_700
- value: 88.967
- - type: mrr_at_1000
- value: 88.967
+ - type: precision_at_20
+ value: 7.217999999999999
+ - type: precision_at_3
+ value: 38.163000000000004
+ - type: precision_at_5
+ value: 25.05
+ - type: recall_at_1
+ value: 71.856
+ - type: recall_at_10
+ value: 96.244
+ - type: recall_at_100
+ value: 99.773
+ - type: recall_at_1000
+ value: 99.99900000000001
+ - type: recall_at_20
+ value: 98.221
+ - type: recall_at_3
+ value: 88.715
+ - type: recall_at_5
+ value: 92.88499999999999
- task:
type: Clustering
dataset:
@@ -2124,7 +2688,7 @@ model-index:
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb
metrics:
- type: v_measure
- value: 59.90388554491155
+ value: 62.91969510127886
- task:
type: Clustering
dataset:
@@ -2132,179 +2696,229 @@ model-index:
name: MTEB RedditClusteringP2P
config: default
split: test
- revision: 282350215ef01743dc01b456c7f5241fa8937f16
+ revision: 385e3cb46b4cfa89021f56c4380204149d0efe33
metrics:
- type: v_measure
- value: 67.64232539036783
+ value: 72.74201090913765
- task:
type: Retrieval
dataset:
- type: scidocs
+ type: mteb/scidocs
name: MTEB SCIDOCS
config: default
split: test
- revision: None
+ revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88
metrics:
- - type: ndcg_at_1
- value: 22.6
- - type: ndcg_at_2
- value: 20.355999999999998
- - type: ndcg_at_3
- value: 18.536
- - type: ndcg_at_5
- value: 16.523
- - type: ndcg_at_7
- value: 17.979
- - type: ndcg_at_10
- value: 19.908
- - type: ndcg_at_20
- value: 22.887
- - type: ndcg_at_30
- value: 24.43
- - type: ndcg_at_50
- value: 25.959
- - type: ndcg_at_70
- value: 26.989
- - type: ndcg_at_100
- value: 27.977
- - type: ndcg_at_200
- value: 29.831000000000003
- - type: ndcg_at_300
- value: 30.787
- - type: ndcg_at_500
- value: 31.974999999999998
- - type: ndcg_at_700
- value: 32.554
- - type: ndcg_at_1000
- value: 33.277
- type: map_at_1
- value: 4.593
- - type: map_at_2
- value: 6.923
- - type: map_at_3
- value: 8.3
- - type: map_at_5
- value: 10.072000000000001
- - type: map_at_7
- value: 10.782
+ value: 5.8229999999999995
- type: map_at_10
- value: 11.72
- - type: map_at_20
- value: 12.838
- - type: map_at_30
- value: 13.257
- - type: map_at_50
- value: 13.569
- - type: map_at_70
- value: 13.733
+ value: 15.152
- type: map_at_100
- value: 13.858999999999998
- - type: map_at_200
- value: 14.018
- - type: map_at_300
- value: 14.072999999999999
- - type: map_at_500
- value: 14.126
- - type: map_at_700
- value: 14.145
+ value: 17.936
- type: map_at_1000
- value: 14.161999999999999
- - type: recall_at_1
- value: 4.593
- - type: recall_at_2
- value: 7.997999999999999
- - type: recall_at_3
- value: 10.563
- - type: recall_at_5
- value: 14.907
- - type: recall_at_7
- value: 17.4
- - type: recall_at_10
- value: 21.18
- - type: recall_at_20
- value: 28.144999999999996
- - type: recall_at_30
- value: 32.462
- - type: recall_at_50
- value: 37.267
- - type: recall_at_70
- value: 40.875
- - type: recall_at_100
- value: 44.641999999999996
- - type: recall_at_200
- value: 52.573
- - type: recall_at_300
- value: 57.089999999999996
- - type: recall_at_500
- value: 63.14300000000001
- - type: recall_at_700
- value: 66.313
- - type: recall_at_1000
- value: 70.458
- - type: precision_at_1
- value: 22.6
- - type: precision_at_2
- value: 19.7
- - type: precision_at_3
- value: 17.333000000000002
- - type: precision_at_5
- value: 14.680000000000001
- - type: precision_at_7
- value: 12.243
- - type: precision_at_10
- value: 10.440000000000001
- - type: precision_at_20
- value: 6.944999999999999
- - type: precision_at_30
- value: 5.333
- - type: precision_at_50
- value: 3.678
- - type: precision_at_70
- value: 2.881
- - type: precision_at_100
- value: 2.2030000000000003
- - type: precision_at_200
- value: 1.295
- - type: precision_at_300
- value: 0.9369999999999999
- - type: precision_at_500
- value: 0.622
- - type: precision_at_700
- value: 0.466
- - type: precision_at_1000
- value: 0.347
+ value: 18.292
+ - type: map_at_20
+ value: 16.526
+ - type: map_at_3
+ value: 10.294
+ - type: map_at_5
+ value: 12.794
- type: mrr_at_1
- value: 22.6
- - type: mrr_at_2
- value: 27.900000000000002
- - type: mrr_at_3
- value: 30.067
- - type: mrr_at_5
- value: 32.207
- - type: mrr_at_7
- value: 33.004
+ value: 28.599999999999998
- type: mrr_at_10
- value: 33.596
- - type: mrr_at_20
- value: 34.268
- - type: mrr_at_30
- value: 34.492
- - type: mrr_at_50
- value: 34.628
- - type: mrr_at_70
- value: 34.681
+ value: 40.68206349206347
- type: mrr_at_100
- value: 34.717
- - type: mrr_at_200
- value: 34.757
- - type: mrr_at_300
- value: 34.768
- - type: mrr_at_500
- value: 34.772
- - type: mrr_at_700
- value: 34.774
+ value: 41.673752995361795
- type: mrr_at_1000
- value: 34.775
+ value: 41.71500072915374
+ - type: mrr_at_20
+ value: 41.28552805166964
+ - type: mrr_at_3
+ value: 36.84999999999998
+ - type: mrr_at_5
+ value: 39.19999999999995
+ - type: nauc_map_at_1000_diff1
+ value: 13.937030183524016
+ - type: nauc_map_at_1000_max
+ value: 29.101253428618957
+ - type: nauc_map_at_100_diff1
+ value: 13.886406833168516
+ - type: nauc_map_at_100_max
+ value: 29.089030812842058
+ - type: nauc_map_at_10_diff1
+ value: 14.40329921271108
+ - type: nauc_map_at_10_max
+ value: 28.00363746647288
+ - type: nauc_map_at_1_diff1
+ value: 23.498080363225856
+ - type: nauc_map_at_1_max
+ value: 22.135916075181783
+ - type: nauc_map_at_20_diff1
+ value: 13.778485906288793
+ - type: nauc_map_at_20_max
+ value: 28.693651813322596
+ - type: nauc_map_at_3_diff1
+ value: 17.60229285176299
+ - type: nauc_map_at_3_max
+ value: 25.413773614488772
+ - type: nauc_map_at_5_diff1
+ value: 15.550704875256194
+ - type: nauc_map_at_5_max
+ value: 26.894948182000878
+ - type: nauc_mrr_at_1000_diff1
+ value: 19.216907937508353
+ - type: nauc_mrr_at_1000_max
+ value: 24.659678077440798
+ - type: nauc_mrr_at_100_diff1
+ value: 19.216060365892424
+ - type: nauc_mrr_at_100_max
+ value: 24.684151414467898
+ - type: nauc_mrr_at_10_diff1
+ value: 18.893433462125277
+ - type: nauc_mrr_at_10_max
+ value: 24.60150470624984
+ - type: nauc_mrr_at_1_diff1
+ value: 23.581793191943625
+ - type: nauc_mrr_at_1_max
+ value: 22.642498223179697
+ - type: nauc_mrr_at_20_diff1
+ value: 19.127783601589428
+ - type: nauc_mrr_at_20_max
+ value: 24.703190799706203
+ - type: nauc_mrr_at_3_diff1
+ value: 18.93114595448597
+ - type: nauc_mrr_at_3_max
+ value: 23.504568736438475
+ - type: nauc_mrr_at_5_diff1
+ value: 18.53230488706395
+ - type: nauc_mrr_at_5_max
+ value: 24.047241648734506
+ - type: nauc_ndcg_at_1000_diff1
+ value: 14.716837800458293
+ - type: nauc_ndcg_at_1000_max
+ value: 30.459415055635038
+ - type: nauc_ndcg_at_100_diff1
+ value: 14.082504373803328
+ - type: nauc_ndcg_at_100_max
+ value: 30.609939275904168
+ - type: nauc_ndcg_at_10_diff1
+ value: 14.06100594429231
+ - type: nauc_ndcg_at_10_max
+ value: 28.41960526781813
+ - type: nauc_ndcg_at_1_diff1
+ value: 23.581793191943625
+ - type: nauc_ndcg_at_1_max
+ value: 22.642498223179697
+ - type: nauc_ndcg_at_20_diff1
+ value: 13.572283433606987
+ - type: nauc_ndcg_at_20_max
+ value: 29.248187501659608
+ - type: nauc_ndcg_at_3_diff1
+ value: 16.747853135741224
+ - type: nauc_ndcg_at_3_max
+ value: 25.225938170933226
+ - type: nauc_ndcg_at_5_diff1
+ value: 15.002619606328727
+ - type: nauc_ndcg_at_5_max
+ value: 26.824444127747
+ - type: nauc_precision_at_1000_diff1
+ value: 6.475682714396218
+ - type: nauc_precision_at_1000_max
+ value: 25.893148901146194
+ - type: nauc_precision_at_100_diff1
+ value: 7.28557838169408
+ - type: nauc_precision_at_100_max
+ value: 28.411459246435584
+ - type: nauc_precision_at_10_diff1
+ value: 8.60582676352359
+ - type: nauc_precision_at_10_max
+ value: 28.531058600175907
+ - type: nauc_precision_at_1_diff1
+ value: 23.581793191943625
+ - type: nauc_precision_at_1_max
+ value: 22.642498223179697
+ - type: nauc_precision_at_20_diff1
+ value: 7.2628155308533335
+ - type: nauc_precision_at_20_max
+ value: 28.292245392834513
+ - type: nauc_precision_at_3_diff1
+ value: 13.67616794325982
+ - type: nauc_precision_at_3_max
+ value: 26.088199174513075
+ - type: nauc_precision_at_5_diff1
+ value: 10.49804462108653
+ - type: nauc_precision_at_5_max
+ value: 27.582324286139148
+ - type: nauc_recall_at_1000_diff1
+ value: 7.3223770822876775
+ - type: nauc_recall_at_1000_max
+ value: 26.257019825355776
+ - type: nauc_recall_at_100_diff1
+ value: 7.129264789675433
+ - type: nauc_recall_at_100_max
+ value: 28.233039459249643
+ - type: nauc_recall_at_10_diff1
+ value: 8.457257641590031
+ - type: nauc_recall_at_10_max
+ value: 28.130585288632325
+ - type: nauc_recall_at_1_diff1
+ value: 23.498080363225856
+ - type: nauc_recall_at_1_max
+ value: 22.135916075181783
+ - type: nauc_recall_at_20_diff1
+ value: 7.077555601648696
+ - type: nauc_recall_at_20_max
+ value: 27.841434750392335
+ - type: nauc_recall_at_3_diff1
+ value: 13.684083144298768
+ - type: nauc_recall_at_3_max
+ value: 25.543860691478663
+ - type: nauc_recall_at_5_diff1
+ value: 10.455816099350045
+ - type: nauc_recall_at_5_max
+ value: 27.290769915982498
+ - type: ndcg_at_1
+ value: 28.599999999999998
+ - type: ndcg_at_10
+ value: 24.866
+ - type: ndcg_at_100
+ value: 34.597
+ - type: ndcg_at_1000
+ value: 39.994
+ - type: ndcg_at_20
+ value: 28.309
+ - type: ndcg_at_3
+ value: 22.749
+ - type: ndcg_at_5
+ value: 20.502000000000002
+ - type: precision_at_1
+ value: 28.599999999999998
+ - type: precision_at_10
+ value: 13.089999999999998
+ - type: precision_at_100
+ value: 2.7119999999999997
+ - type: precision_at_1000
+ value: 0.39899999999999997
+ - type: precision_at_20
+ value: 8.53
+ - type: precision_at_3
+ value: 21.099999999999998
+ - type: precision_at_5
+ value: 18.22
+ - type: recall_at_1
+ value: 5.8229999999999995
+ - type: recall_at_10
+ value: 26.522000000000002
+ - type: recall_at_100
+ value: 55.003
+ - type: recall_at_1000
+ value: 80.977
+ - type: recall_at_20
+ value: 34.618
+ - type: recall_at_3
+ value: 12.848
+ - type: recall_at_5
+ value: 18.477
- task:
type: STS
dataset:
@@ -2312,20 +2926,20 @@ model-index:
name: MTEB SICK-R
config: default
split: test
- revision: a6ea5a8cab320b040a23452cc28066d9beae2cee
+ revision: 20a6d6f312dd54037fe07a32d58e5e168867909d
metrics:
- type: cos_sim_pearson
- value: 86.90122745229677
+ value: 80.72562067620224
- type: cos_sim_spearman
- value: 82.92294737327579
+ value: 77.00710192931953
- type: euclidean_pearson
- value: 84.08979655773187
+ value: 78.65843289108192
- type: euclidean_spearman
- value: 82.92294657285412
+ value: 77.00710077709005
- type: manhattan_pearson
- value: 84.09347480531832
+ value: 78.48859522905846
- type: manhattan_spearman
- value: 82.91564613948087
+ value: 76.8213740840866
- task:
type: STS
dataset:
@@ -2336,17 +2950,17 @@ model-index:
revision: a0d554a64d88156834ff5ae9920b964011b16384
metrics:
- type: cos_sim_pearson
- value: 87.01218713698583
+ value: 81.15015325911659
- type: cos_sim_spearman
- value: 79.46865215168464
+ value: 75.67268325741222
- type: euclidean_pearson
- value: 83.22621889891909
+ value: 75.54004763633206
- type: euclidean_spearman
- value: 79.46853821709514
+ value: 75.67262179635058
- type: manhattan_pearson
- value: 83.69962580788805
+ value: 75.80681616893116
- type: manhattan_spearman
- value: 79.9561593356932
+ value: 75.93721016401406
- task:
type: STS
dataset:
@@ -2357,17 +2971,17 @@ model-index:
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca
metrics:
- type: cos_sim_pearson
- value: 88.98438696342964
+ value: 81.71651874476737
- type: cos_sim_spearman
- value: 89.15419511870839
+ value: 82.39667472464997
- type: euclidean_pearson
- value: 88.49646141802894
+ value: 82.28256504757712
- type: euclidean_spearman
- value: 89.15419503946019
+ value: 82.39663674872656
- type: manhattan_pearson
- value: 88.6420585616327
+ value: 82.3192873176068
- type: manhattan_spearman
- value: 89.42648950757743
+ value: 82.41915252757059
- task:
type: STS
dataset:
@@ -2378,17 +2992,17 @@ model-index:
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375
metrics:
- type: cos_sim_pearson
- value: 87.30772547759544
+ value: 81.222967367593
- type: cos_sim_spearman
- value: 84.93199878424691
+ value: 79.92685877403252
- type: euclidean_pearson
- value: 86.16266630395455
+ value: 79.95053542861498
- type: euclidean_spearman
- value: 84.93198798543634
+ value: 79.9268858850991
- type: manhattan_pearson
- value: 86.14285723189803
+ value: 79.90485851323321
- type: manhattan_spearman
- value: 85.0361672522687
+ value: 79.93878025669312
- task:
type: STS
dataset:
@@ -2399,17 +3013,17 @@ model-index:
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3
metrics:
- type: cos_sim_pearson
- value: 90.21342071197127
+ value: 85.27539130156643
- type: cos_sim_spearman
- value: 90.7407512744838
+ value: 85.81645767911826
- type: euclidean_pearson
- value: 90.1517933113061
+ value: 85.5488615685444
- type: euclidean_spearman
- value: 90.74075125431919
+ value: 85.81647022566916
- type: manhattan_pearson
- value: 90.17963034676193
+ value: 85.6358149547879
- type: manhattan_spearman
- value: 90.88999275865135
+ value: 85.96347118567043
- task:
type: STS
dataset:
@@ -2420,59 +3034,59 @@ model-index:
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513
metrics:
- type: cos_sim_pearson
- value: 86.82518054100498
+ value: 83.43727336154858
- type: cos_sim_spearman
- value: 87.81570533154735
+ value: 84.50468882202796
- type: euclidean_pearson
- value: 86.91684561573618
+ value: 83.23576727105372
- type: euclidean_spearman
- value: 87.81570533154735
+ value: 84.50468882202796
- type: manhattan_pearson
- value: 86.98311935744032
+ value: 83.28843314503176
- type: manhattan_spearman
- value: 87.9594667151966
+ value: 84.60383766214322
- task:
type: STS
dataset:
type: mteb/sts17-crosslingual-sts
- name: MTEB STS17 (en-en)
- config: en-en
+ name: MTEB STS17
+ config: default
split: test
- revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
+ revision: faeb762787bd10488a50c8b5be4a3b82e411949c
metrics:
- type: cos_sim_pearson
- value: 92.09578436612053
+ value: 88.86589365166874
- type: cos_sim_spearman
- value: 92.01519349090438
+ value: 88.93117996163835
- type: euclidean_pearson
- value: 92.07113635890894
+ value: 89.12271565981082
- type: euclidean_spearman
- value: 92.01519349090438
+ value: 88.93117996163835
- type: manhattan_pearson
- value: 91.89343820765625
+ value: 88.94419759325545
- type: manhattan_spearman
- value: 91.7443476810177
+ value: 88.63073561731899
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
- name: MTEB STS22 (en)
- config: en
+ name: MTEB STS22
+ config: default
split: test
- revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
+ revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
metrics:
- type: cos_sim_pearson
- value: 69.29997751464549
+ value: 67.96578378422929
- type: cos_sim_spearman
- value: 68.36425436812782
+ value: 67.10257461424345
- type: euclidean_pearson
- value: 69.81381677661783
+ value: 67.51317866195149
- type: euclidean_spearman
- value: 68.36425436812782
+ value: 67.10257461424345
- type: manhattan_pearson
- value: 69.92823397008026
+ value: 67.74940912013754
- type: manhattan_spearman
- value: 68.35770640039254
+ value: 67.46694183937207
- task:
type: STS
dataset:
@@ -2483,17 +3097,17 @@ model-index:
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831
metrics:
- type: cos_sim_pearson
- value: 88.39126315452359
+ value: 83.55433725920493
- type: cos_sim_spearman
- value: 88.99708463265337
+ value: 83.60373857254014
- type: euclidean_pearson
- value: 88.60793820038607
+ value: 83.08086082334839
- type: euclidean_spearman
- value: 88.99708463265337
+ value: 83.6036864776559
- type: manhattan_pearson
- value: 88.69860633571047
+ value: 83.2232267589246
- type: manhattan_spearman
- value: 89.20094593888012
+ value: 83.78923946962664
- task:
type: Reranking
dataset:
@@ -2504,178 +3118,236 @@ model-index:
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab
metrics:
- type: map
- value: 86.58028062818582
+ value: 87.28566757174322
- type: mrr
- value: 96.53586790841693
+ value: 96.63827639317836
+ - type: nAUC_map_diff1
+ value: -1.5076360579080446
+ - type: nAUC_map_max
+ value: 53.05146289659708
+ - type: nAUC_mrr_diff1
+ value: 42.43768025278795
+ - type: nAUC_mrr_max
+ value: 89.70532867621158
- task:
type: Retrieval
dataset:
- type: scifact
+ type: mteb/scifact
name: MTEB SciFact
config: default
split: test
- revision: None
+ revision: 0228b52cf27578f30900b9e5271d331663a030d7
metrics:
- - type: ndcg_at_1
- value: 66.333
- - type: ndcg_at_2
- value: 70.655
- - type: ndcg_at_3
- value: 72.801
- - type: ndcg_at_5
- value: 75.793
- - type: ndcg_at_7
- value: 76.946
- - type: ndcg_at_10
- value: 77.66199999999999
- - type: ndcg_at_20
- value: 78.786
- - type: ndcg_at_30
- value: 79.066
- - type: ndcg_at_50
- value: 79.255
- - type: ndcg_at_70
- value: 79.423
- - type: ndcg_at_100
- value: 79.476
- - type: ndcg_at_200
- value: 79.65299999999999
- - type: ndcg_at_300
- value: 79.696
- - type: ndcg_at_500
- value: 79.73599999999999
- - type: ndcg_at_700
- value: 79.77199999999999
- - type: ndcg_at_1000
- value: 79.77199999999999
- type: map_at_1
- value: 63.383
- - type: map_at_2
- value: 68.144
- - type: map_at_3
- value: 70.19800000000001
- - type: map_at_5
- value: 72.38
- - type: map_at_7
- value: 72.955
+ value: 70.661
- type: map_at_10
- value: 73.312
- - type: map_at_20
- value: 73.678
- - type: map_at_30
- value: 73.72800000000001
- - type: map_at_50
- value: 73.75500000000001
- - type: map_at_70
- value: 73.771
+ value: 82.051
- type: map_at_100
- value: 73.776
- - type: map_at_200
- value: 73.783
- - type: map_at_300
- value: 73.784
- - type: map_at_500
- value: 73.785
- - type: map_at_700
- value: 73.786
+ value: 82.162
- type: map_at_1000
- value: 73.786
- - type: recall_at_1
- value: 63.383
- - type: recall_at_2
- value: 72.283
- - type: recall_at_3
- value: 77.183
- - type: recall_at_5
- value: 84.56099999999999
- - type: recall_at_7
- value: 87.67200000000001
- - type: recall_at_10
- value: 89.822
- - type: recall_at_20
- value: 94
- - type: recall_at_30
- value: 95.333
- - type: recall_at_50
- value: 96.333
- - type: recall_at_70
- value: 97.333
- - type: recall_at_100
- value: 97.667
- - type: recall_at_200
- value: 99
- - type: recall_at_300
- value: 99.333
- - type: recall_at_500
- value: 99.667
- - type: recall_at_700
- value: 100
- - type: recall_at_1000
- value: 100
+ value: 82.167
+ - type: map_at_20
+ value: 82.122
+ - type: map_at_3
+ value: 79.919
+ - type: map_at_5
+ value: 81.368
+ - type: mrr_at_1
+ value: 74.33333333333333
+ - type: mrr_at_10
+ value: 82.98452380952381
+ - type: mrr_at_100
+ value: 83.09512420633841
+ - type: mrr_at_1000
+ value: 83.10026279387446
+ - type: mrr_at_20
+ value: 83.05460927960928
+ - type: mrr_at_3
+ value: 81.8888888888889
+ - type: mrr_at_5
+ value: 82.65555555555557
+ - type: nauc_map_at_1000_diff1
+ value: 74.23493814910577
+ - type: nauc_map_at_1000_max
+ value: 53.80760307987988
+ - type: nauc_map_at_100_diff1
+ value: 74.23892667859884
+ - type: nauc_map_at_100_max
+ value: 53.8218538604195
+ - type: nauc_map_at_10_diff1
+ value: 74.0962352399456
+ - type: nauc_map_at_10_max
+ value: 53.99920687179696
+ - type: nauc_map_at_1_diff1
+ value: 76.44442261798511
+ - type: nauc_map_at_1_max
+ value: 44.51235822441656
+ - type: nauc_map_at_20_diff1
+ value: 74.2585238520073
+ - type: nauc_map_at_20_max
+ value: 53.90125243656952
+ - type: nauc_map_at_3_diff1
+ value: 74.89158646614453
+ - type: nauc_map_at_3_max
+ value: 49.41963788438226
+ - type: nauc_map_at_5_diff1
+ value: 73.8209929680174
+ - type: nauc_map_at_5_max
+ value: 52.98176980743803
+ - type: nauc_mrr_at_1000_diff1
+ value: 73.26418373110498
+ - type: nauc_mrr_at_1000_max
+ value: 56.29463433767135
+ - type: nauc_mrr_at_100_diff1
+ value: 73.26866985211069
+ - type: nauc_mrr_at_100_max
+ value: 56.3086756476332
+ - type: nauc_mrr_at_10_diff1
+ value: 73.12672927850232
+ - type: nauc_mrr_at_10_max
+ value: 56.47596127739652
+ - type: nauc_mrr_at_1_diff1
+ value: 75.37819545277598
+ - type: nauc_mrr_at_1_max
+ value: 53.29021506771669
+ - type: nauc_mrr_at_20_diff1
+ value: 73.29166408154734
+ - type: nauc_mrr_at_20_max
+ value: 56.38499626340773
+ - type: nauc_mrr_at_3_diff1
+ value: 72.95802894910716
+ - type: nauc_mrr_at_3_max
+ value: 56.06917546432742
+ - type: nauc_mrr_at_5_diff1
+ value: 72.68856991547275
+ - type: nauc_mrr_at_5_max
+ value: 56.4504016425308
+ - type: nauc_ndcg_at_1000_diff1
+ value: 73.43694724531103
+ - type: nauc_ndcg_at_1000_max
+ value: 55.74280339933835
+ - type: nauc_ndcg_at_100_diff1
+ value: 73.47499591728393
+ - type: nauc_ndcg_at_100_max
+ value: 56.1686438223656
+ - type: nauc_ndcg_at_10_diff1
+ value: 72.80109276813523
+ - type: nauc_ndcg_at_10_max
+ value: 57.1765306847374
+ - type: nauc_ndcg_at_1_diff1
+ value: 75.37819545277598
+ - type: nauc_ndcg_at_1_max
+ value: 53.29021506771669
+ - type: nauc_ndcg_at_20_diff1
+ value: 73.49077190247995
+ - type: nauc_ndcg_at_20_max
+ value: 56.758562898067076
+ - type: nauc_ndcg_at_3_diff1
+ value: 73.4565027782411
+ - type: nauc_ndcg_at_3_max
+ value: 53.68675653729513
+ - type: nauc_ndcg_at_5_diff1
+ value: 71.94647594667556
+ - type: nauc_ndcg_at_5_max
+ value: 55.780093959984825
+ - type: nauc_precision_at_1000_diff1
+ value: -35.560325552276225
+ - type: nauc_precision_at_1000_max
+ value: 15.721037749478894
+ - type: nauc_precision_at_100_diff1
+ value: -30.911390903880072
+ - type: nauc_precision_at_100_max
+ value: 19.104750986631856
+ - type: nauc_precision_at_10_diff1
+ value: -21.747032531154677
+ - type: nauc_precision_at_10_max
+ value: 25.16531983542022
+ - type: nauc_precision_at_1_diff1
+ value: 75.37819545277598
+ - type: nauc_precision_at_1_max
+ value: 53.29021506771669
+ - type: nauc_precision_at_20_diff1
+ value: -24.171482377268557
+ - type: nauc_precision_at_20_max
+ value: 23.052021983026588
+ - type: nauc_precision_at_3_diff1
+ value: 12.687710012191802
+ - type: nauc_precision_at_3_max
+ value: 36.671474025394744
+ - type: nauc_precision_at_5_diff1
+ value: -10.650013965923193
+ - type: nauc_precision_at_5_max
+ value: 33.35727491587196
+ - type: nauc_recall_at_1000_diff1
+ value: nan
+ - type: nauc_recall_at_1000_max
+ value: nan
+ - type: nauc_recall_at_100_diff1
+ value: 74.24525365701795
+ - type: nauc_recall_at_100_max
+ value: 95.64270152505469
+ - type: nauc_recall_at_10_diff1
+ value: 63.46659876071664
+ - type: nauc_recall_at_10_max
+ value: 83.42670401493947
+ - type: nauc_recall_at_1_diff1
+ value: 76.44442261798511
+ - type: nauc_recall_at_1_max
+ value: 44.51235822441656
+ - type: nauc_recall_at_20_diff1
+ value: 72.43230625583546
+ - type: nauc_recall_at_20_max
+ value: 85.23576097105469
+ - type: nauc_recall_at_3_diff1
+ value: 68.64536013381606
+ - type: nauc_recall_at_3_max
+ value: 49.51021446817795
+ - type: nauc_recall_at_5_diff1
+ value: 59.64690270534777
+ - type: nauc_recall_at_5_max
+ value: 63.634671446284706
+ - type: ndcg_at_1
+ value: 74.333
+ - type: ndcg_at_10
+ value: 85.914
+ - type: ndcg_at_100
+ value: 86.473
+ - type: ndcg_at_1000
+ value: 86.602
+ - type: ndcg_at_20
+ value: 86.169
+ - type: ndcg_at_3
+ value: 83.047
+ - type: ndcg_at_5
+ value: 84.72
- type: precision_at_1
- value: 66.333
- - type: precision_at_2
- value: 38.667
- - type: precision_at_3
- value: 28.111000000000004
- - type: precision_at_5
- value: 18.933
- - type: precision_at_7
- value: 14.094999999999999
+ value: 74.333
- type: precision_at_10
- value: 10.167
- - type: precision_at_20
- value: 5.35
- - type: precision_at_30
- value: 3.611
- - type: precision_at_50
- value: 2.1870000000000003
- - type: precision_at_70
- value: 1.576
+ value: 10.933
- type: precision_at_100
- value: 1.107
- - type: precision_at_200
- value: 0.5599999999999999
- - type: precision_at_300
- value: 0.374
- - type: precision_at_500
- value: 0.22499999999999998
- - type: precision_at_700
- value: 0.161
+ value: 1.1199999999999999
- type: precision_at_1000
value: 0.11299999999999999
- - type: mrr_at_1
- value: 66.333
- - type: mrr_at_2
- value: 70.833
- - type: mrr_at_3
- value: 72.167
- - type: mrr_at_5
- value: 73.6
- - type: mrr_at_7
- value: 74.084
- - type: mrr_at_10
- value: 74.283
- - type: mrr_at_20
- value: 74.54499999999999
- - type: mrr_at_30
- value: 74.59599999999999
- - type: mrr_at_50
- value: 74.622
- - type: mrr_at_70
- value: 74.639
- - type: mrr_at_100
- value: 74.643
- - type: mrr_at_200
- value: 74.65
- - type: mrr_at_300
- value: 74.652
- - type: mrr_at_500
- value: 74.653
- - type: mrr_at_700
- value: 74.653
- - type: mrr_at_1000
- value: 74.653
+ - type: precision_at_20
+ value: 5.5169999999999995
+ - type: precision_at_3
+ value: 32.444
+ - type: precision_at_5
+ value: 20.8
+ - type: recall_at_1
+ value: 70.661
+ - type: recall_at_10
+ value: 96.333
+ - type: recall_at_100
+ value: 99.0
+ - type: recall_at_1000
+ value: 100.0
+ - type: recall_at_20
+ value: 97.333
+ - type: recall_at_3
+ value: 88.64999999999999
+ - type: recall_at_5
+ value: 93.089
- task:
type: PairClassification
dataset:
@@ -2686,51 +3358,51 @@ model-index:
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46
metrics:
- type: cos_sim_accuracy
- value: 99.84554455445544
+ value: 99.89108910891089
- type: cos_sim_ap
- value: 96.31178339136798
+ value: 97.61815451002174
- type: cos_sim_f1
- value: 92.1921921921922
+ value: 94.51097804391219
- type: cos_sim_precision
- value: 92.28456913827655
+ value: 94.32270916334662
- type: cos_sim_recall
- value: 92.10000000000001
+ value: 94.69999999999999
- type: dot_accuracy
- value: 99.84554455445544
+ value: 99.89108910891089
- type: dot_ap
- value: 96.31178339136797
+ value: 97.61815451002175
- type: dot_f1
- value: 92.1921921921922
+ value: 94.51097804391219
- type: dot_precision
- value: 92.28456913827655
+ value: 94.32270916334662
- type: dot_recall
- value: 92.10000000000001
+ value: 94.69999999999999
- type: euclidean_accuracy
- value: 99.84554455445544
+ value: 99.89108910891089
- type: euclidean_ap
- value: 96.31178339136798
+ value: 97.61815534251431
- type: euclidean_f1
- value: 92.1921921921922
+ value: 94.51097804391219
- type: euclidean_precision
- value: 92.28456913827655
+ value: 94.32270916334662
- type: euclidean_recall
- value: 92.10000000000001
+ value: 94.69999999999999
- type: manhattan_accuracy
- value: 99.84752475247525
+ value: 99.8940594059406
- type: manhattan_ap
- value: 96.4591954606088
+ value: 97.66124472227202
- type: manhattan_f1
- value: 92.25352112676056
+ value: 94.65267366316841
- type: manhattan_precision
- value: 92.81376518218623
+ value: 94.60539460539461
- type: manhattan_recall
- value: 91.7
+ value: 94.69999999999999
- type: max_accuracy
- value: 99.84752475247525
+ value: 99.8940594059406
- type: max_ap
- value: 96.4591954606088
+ value: 97.66124472227202
- type: max_f1
- value: 92.25352112676056
+ value: 94.65267366316841
- task:
type: Clustering
dataset:
@@ -2741,7 +3413,7 @@ model-index:
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259
metrics:
- type: v_measure
- value: 74.24659759283294
+ value: 76.482776391195
- task:
type: Clustering
dataset:
@@ -2752,7 +3424,7 @@ model-index:
revision: 815ca46b2622cec33ccafc3735d572c266efdb44
metrics:
- type: v_measure
- value: 46.77690051260451
+ value: 48.29023235124473
- task:
type: Reranking
dataset:
@@ -2763,9 +3435,17 @@ model-index:
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69
metrics:
- type: map
- value: 55.68436757803185
+ value: 55.3190739691685
- type: mrr
- value: 56.82157711569475
+ value: 56.40441972243442
+ - type: nAUC_map_diff1
+ value: 41.825825138514354
+ - type: nAUC_map_max
+ value: 11.323603372075258
+ - type: nAUC_mrr_diff1
+ value: 41.85341549688017
+ - type: nAUC_mrr_max
+ value: 12.118291862543504
- task:
type: Summarization
dataset:
@@ -2776,351 +3456,451 @@ model-index:
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c
metrics:
- type: cos_sim_pearson
- value: 31.652482405629843
+ value: 31.98570594378664
- type: cos_sim_spearman
- value: 31.16341822347735
+ value: 30.712965330802174
- type: dot_pearson
- value: 31.652479892699837
+ value: 31.98570540209124
- type: dot_spearman
- value: 31.16341822347735
+ value: 30.712965330802174
- task:
type: Retrieval
dataset:
- type: trec-covid
+ type: mteb/trec-covid
name: MTEB TRECCOVID
config: default
split: test
- revision: None
+ revision: bb9466bac8153a0349341eb1b22e06409e78ef4e
metrics:
- - type: ndcg_at_1
- value: 92
- - type: ndcg_at_2
- value: 90.839
- - type: ndcg_at_3
- value: 90.642
- - type: ndcg_at_5
- value: 90.348
- - type: ndcg_at_7
- value: 89.015
- - type: ndcg_at_10
- value: 87.599
- - type: ndcg_at_20
- value: 84.434
- - type: ndcg_at_30
- value: 81.655
- - type: ndcg_at_50
- value: 77.278
- - type: ndcg_at_70
- value: 73.957
- - type: ndcg_at_100
- value: 69.56
- - type: ndcg_at_200
- value: 60.724000000000004
- - type: ndcg_at_300
- value: 57.245000000000005
- - type: ndcg_at_500
- value: 56.316
- - type: ndcg_at_700
- value: 58.399
- - type: ndcg_at_1000
- value: 62.21600000000001
- type: map_at_1
- value: 0.247
- - type: map_at_2
- value: 0.488
- - type: map_at_3
- value: 0.7230000000000001
- - type: map_at_5
- value: 1.204
- - type: map_at_7
- value: 1.6500000000000001
+ value: 0.25
- type: map_at_10
- value: 2.292
- - type: map_at_20
- value: 4.274
- - type: map_at_30
- value: 6.027
- - type: map_at_50
- value: 9.083
- - type: map_at_70
- value: 11.751000000000001
+ value: 2.2640000000000002
- type: map_at_100
- value: 14.912
- - type: map_at_200
- value: 22.213
- - type: map_at_300
- value: 26.667999999999996
- - type: map_at_500
- value: 31.556
- - type: map_at_700
- value: 34.221000000000004
+ value: 14.447
- type: map_at_1000
- value: 36.443999999999996
- - type: recall_at_1
- value: 0.247
- - type: recall_at_2
- value: 0.49899999999999994
- - type: recall_at_3
- value: 0.742
- - type: recall_at_5
- value: 1.247
- - type: recall_at_7
- value: 1.722
- - type: recall_at_10
- value: 2.405
- - type: recall_at_20
- value: 4.583
- - type: recall_at_30
- value: 6.587999999999999
- - type: recall_at_50
- value: 10.188
- - type: recall_at_70
- value: 13.496
- - type: recall_at_100
- value: 17.578
- - type: recall_at_200
- value: 28.158
- - type: recall_at_300
- value: 35.532000000000004
- - type: recall_at_500
- value: 45.31
- - type: recall_at_700
- value: 51.822
- - type: recall_at_1000
- value: 58.53
- - type: precision_at_1
- value: 96
- - type: precision_at_2
- value: 96
- - type: precision_at_3
- value: 95.333
- - type: precision_at_5
- value: 94.8
- - type: precision_at_7
- value: 93.429
- - type: precision_at_10
- value: 91.4
- - type: precision_at_20
- value: 87.7
- - type: precision_at_30
- value: 84.867
- - type: precision_at_50
- value: 80.24
- - type: precision_at_70
- value: 76.371
- - type: precision_at_100
- value: 71.08
- - type: precision_at_200
- value: 59.4
- - type: precision_at_300
- value: 51.459999999999994
- - type: precision_at_500
- value: 40.644000000000005
- - type: precision_at_700
- value: 33.889
- - type: precision_at_1000
- value: 27.250000000000004
+ value: 35.452
+ - type: map_at_20
+ value: 4.163
+ - type: map_at_3
+ value: 0.715
+ - type: map_at_5
+ value: 1.1780000000000002
- type: mrr_at_1
- value: 96
- - type: mrr_at_2
- value: 98
- - type: mrr_at_3
- value: 98
- - type: mrr_at_5
- value: 98
- - type: mrr_at_7
- value: 98
+ value: 94.0
- type: mrr_at_10
- value: 98
- - type: mrr_at_20
- value: 98
- - type: mrr_at_30
- value: 98
- - type: mrr_at_50
- value: 98
- - type: mrr_at_70
- value: 98
+ value: 96.66666666666667
- type: mrr_at_100
- value: 98
- - type: mrr_at_200
- value: 98
- - type: mrr_at_300
- value: 98
- - type: mrr_at_500
- value: 98
- - type: mrr_at_700
- value: 98
+ value: 96.66666666666667
- type: mrr_at_1000
- value: 98
+ value: 96.66666666666667
+ - type: mrr_at_20
+ value: 96.66666666666667
+ - type: mrr_at_3
+ value: 96.66666666666667
+ - type: mrr_at_5
+ value: 96.66666666666667
+ - type: nauc_map_at_1000_diff1
+ value: -5.9732852648987205
+ - type: nauc_map_at_1000_max
+ value: 35.50616065092173
+ - type: nauc_map_at_100_diff1
+ value: 12.825728128957033
+ - type: nauc_map_at_100_max
+ value: 15.949290291379766
+ - type: nauc_map_at_10_diff1
+ value: 27.017295749131076
+ - type: nauc_map_at_10_max
+ value: 1.5220901118362347
+ - type: nauc_map_at_1_diff1
+ value: 25.02750049169676
+ - type: nauc_map_at_1_max
+ value: 0.5677927795699148
+ - type: nauc_map_at_20_diff1
+ value: 24.347002216586475
+ - type: nauc_map_at_20_max
+ value: 5.243857803083133
+ - type: nauc_map_at_3_diff1
+ value: 25.816546894429298
+ - type: nauc_map_at_3_max
+ value: -1.6242364528764006
+ - type: nauc_map_at_5_diff1
+ value: 26.02737843714819
+ - type: nauc_map_at_5_max
+ value: -0.20821382476722475
+ - type: nauc_mrr_at_1000_diff1
+ value: -3.9542483660123176
+ - type: nauc_mrr_at_1000_max
+ value: 86.62464985994423
+ - type: nauc_mrr_at_100_diff1
+ value: -3.9542483660123176
+ - type: nauc_mrr_at_100_max
+ value: 86.62464985994423
+ - type: nauc_mrr_at_10_diff1
+ value: -3.9542483660123176
+ - type: nauc_mrr_at_10_max
+ value: 86.62464985994423
+ - type: nauc_mrr_at_1_diff1
+ value: -10.550887021475235
+ - type: nauc_mrr_at_1_max
+ value: 85.13849984438222
+ - type: nauc_mrr_at_20_diff1
+ value: -3.9542483660123176
+ - type: nauc_mrr_at_20_max
+ value: 86.62464985994423
+ - type: nauc_mrr_at_3_diff1
+ value: -3.9542483660123176
+ - type: nauc_mrr_at_3_max
+ value: 86.62464985994423
+ - type: nauc_mrr_at_5_diff1
+ value: -3.9542483660123176
+ - type: nauc_mrr_at_5_max
+ value: 86.62464985994423
+ - type: nauc_ndcg_at_1000_diff1
+ value: -2.5807588587359565
+ - type: nauc_ndcg_at_1000_max
+ value: 36.5814781759383
+ - type: nauc_ndcg_at_100_diff1
+ value: -24.616560982160962
+ - type: nauc_ndcg_at_100_max
+ value: 35.70669109885634
+ - type: nauc_ndcg_at_10_diff1
+ value: -27.95472757545005
+ - type: nauc_ndcg_at_10_max
+ value: 38.321003412946155
+ - type: nauc_ndcg_at_1_diff1
+ value: -4.090802987861885
+ - type: nauc_ndcg_at_1_max
+ value: 70.7399626517274
+ - type: nauc_ndcg_at_20_diff1
+ value: -36.229932610340036
+ - type: nauc_ndcg_at_20_max
+ value: 40.77842743464363
+ - type: nauc_ndcg_at_3_diff1
+ value: -14.881738406653747
+ - type: nauc_ndcg_at_3_max
+ value: 37.376017601422326
+ - type: nauc_ndcg_at_5_diff1
+ value: -21.780436044004624
+ - type: nauc_ndcg_at_5_max
+ value: 34.8771855807311
+ - type: nauc_precision_at_1000_diff1
+ value: -30.91477904447032
+ - type: nauc_precision_at_1000_max
+ value: 33.57647416618872
+ - type: nauc_precision_at_100_diff1
+ value: -23.4629541500978
+ - type: nauc_precision_at_100_max
+ value: 33.67654215432449
+ - type: nauc_precision_at_10_diff1
+ value: -16.3738640229384
+ - type: nauc_precision_at_10_max
+ value: 40.64610973416934
+ - type: nauc_precision_at_1_diff1
+ value: -10.550887021475235
+ - type: nauc_precision_at_1_max
+ value: 85.13849984438222
+ - type: nauc_precision_at_20_diff1
+ value: -28.140856454340984
+ - type: nauc_precision_at_20_max
+ value: 45.91605091028829
+ - type: nauc_precision_at_3_diff1
+ value: -16.543438077633727
+ - type: nauc_precision_at_3_max
+ value: 37.197429803714506
+ - type: nauc_precision_at_5_diff1
+ value: -16.06909604670385
+ - type: nauc_precision_at_5_max
+ value: 42.614560285767524
+ - type: nauc_recall_at_1000_diff1
+ value: 3.7664293917121894
+ - type: nauc_recall_at_1000_max
+ value: 30.305451624799655
+ - type: nauc_recall_at_100_diff1
+ value: 23.081787470102572
+ - type: nauc_recall_at_100_max
+ value: 6.655141400910628
+ - type: nauc_recall_at_10_diff1
+ value: 31.903839603714584
+ - type: nauc_recall_at_10_max
+ value: -2.4370705738706584
+ - type: nauc_recall_at_1_diff1
+ value: 25.02750049169676
+ - type: nauc_recall_at_1_max
+ value: 0.5677927795699148
+ - type: nauc_recall_at_20_diff1
+ value: 28.70508508058853
+ - type: nauc_recall_at_20_max
+ value: 0.45751771491989157
+ - type: nauc_recall_at_3_diff1
+ value: 27.28040405266442
+ - type: nauc_recall_at_3_max
+ value: -4.203535404137551
+ - type: nauc_recall_at_5_diff1
+ value: 28.56501894814338
+ - type: nauc_recall_at_5_max
+ value: -2.885460743520312
+ - type: ndcg_at_1
+ value: 92.0
+ - type: ndcg_at_10
+ value: 87.26899999999999
+ - type: ndcg_at_100
+ value: 68.586
+ - type: ndcg_at_1000
+ value: 61.056999999999995
+ - type: ndcg_at_20
+ value: 83.452
+ - type: ndcg_at_3
+ value: 90.11200000000001
+ - type: ndcg_at_5
+ value: 89.103
+ - type: precision_at_1
+ value: 94.0
+ - type: precision_at_10
+ value: 91.2
+ - type: precision_at_100
+ value: 70.12
+ - type: precision_at_1000
+ value: 26.773999999999997
+ - type: precision_at_20
+ value: 87.3
+ - type: precision_at_3
+ value: 92.667
+ - type: precision_at_5
+ value: 92.4
+ - type: recall_at_1
+ value: 0.25
+ - type: recall_at_10
+ value: 2.3970000000000002
+ - type: recall_at_100
+ value: 17.233999999999998
+ - type: recall_at_1000
+ value: 57.879000000000005
+ - type: recall_at_20
+ value: 4.508
+ - type: recall_at_3
+ value: 0.734
+ - type: recall_at_5
+ value: 1.2269999999999999
- task:
type: Retrieval
dataset:
- type: webis-touche2020
+ type: mteb/touche2020
name: MTEB Touche2020
config: default
split: test
- revision: None
+ revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f
metrics:
- - type: ndcg_at_1
- value: 43.878
- - type: ndcg_at_2
- value: 37.956
- - type: ndcg_at_3
- value: 35.053
- - type: ndcg_at_5
- value: 32.59
- - type: ndcg_at_7
- value: 30.226
- - type: ndcg_at_10
- value: 29.005
- - type: ndcg_at_20
- value: 30.11
- - type: ndcg_at_30
- value: 32.019999999999996
- - type: ndcg_at_50
- value: 34.354
- - type: ndcg_at_70
- value: 36.665
- - type: ndcg_at_100
- value: 38.888
- - type: ndcg_at_200
- value: 43.435
- - type: ndcg_at_300
- value: 45.795
- - type: ndcg_at_500
- value: 48.699999999999996
- - type: ndcg_at_700
- value: 50.242
- - type: ndcg_at_1000
- value: 51.529
- type: map_at_1
- value: 3.521
- - type: map_at_2
- value: 5.309
- - type: map_at_3
- value: 6.576
- - type: map_at_5
- value: 8.97
- - type: map_at_7
- value: 10.194
+ value: 2.806
- type: map_at_10
- value: 11.949
- - type: map_at_20
- value: 14.686
- - type: map_at_30
- value: 15.8
- - type: map_at_50
- value: 16.59
- - type: map_at_70
- value: 17.2
+ value: 11.369
- type: map_at_100
- value: 17.765
- - type: map_at_200
- value: 18.636
- - type: map_at_300
- value: 18.972
- - type: map_at_500
- value: 19.301
- - type: map_at_700
- value: 19.445
+ value: 17.791
- type: map_at_1000
- value: 19.546
- - type: recall_at_1
- value: 3.521
- - type: recall_at_2
- value: 5.848
- - type: recall_at_3
- value: 7.657
- - type: recall_at_5
- value: 11.368
- - type: recall_at_7
- value: 13.748
- - type: recall_at_10
- value: 18.061
- - type: recall_at_20
- value: 26.844
- - type: recall_at_30
- value: 31.186000000000003
- - type: recall_at_50
- value: 35.951
- - type: recall_at_70
- value: 40.961999999999996
- - type: recall_at_100
- value: 46.743
- - type: recall_at_200
- value: 58.483
- - type: recall_at_300
- value: 65.973
- - type: recall_at_500
- value: 75.233
- - type: recall_at_700
- value: 80.472
- - type: recall_at_1000
- value: 85.02
- - type: precision_at_1
- value: 46.939
- - type: precision_at_2
- value: 38.775999999999996
- - type: precision_at_3
- value: 34.694
- - type: precision_at_5
- value: 31.429000000000002
- - type: precision_at_7
- value: 27.697
- - type: precision_at_10
- value: 24.490000000000002
- - type: precision_at_20
- value: 18.776
- - type: precision_at_30
- value: 15.034
- - type: precision_at_50
- value: 10.857
- - type: precision_at_70
- value: 9.096
- - type: precision_at_100
- value: 7.51
- - type: precision_at_200
- value: 4.929
- - type: precision_at_300
- value: 3.7760000000000002
- - type: precision_at_500
- value: 2.6780000000000004
- - type: precision_at_700
- value: 2.085
- - type: precision_at_1000
- value: 1.5709999999999997
+ value: 19.363
+ - type: map_at_20
+ value: 14.038999999999998
+ - type: map_at_3
+ value: 5.817
+ - type: map_at_5
+ value: 8.331
- type: mrr_at_1
- value: 46.939
- - type: mrr_at_2
- value: 55.102
- - type: mrr_at_3
- value: 57.823
- - type: mrr_at_5
- value: 60.68
- - type: mrr_at_7
- value: 60.972
+ value: 36.734693877551024
- type: mrr_at_10
- value: 61.199000000000005
- - type: mrr_at_20
- value: 61.831
- - type: mrr_at_30
- value: 61.831
- - type: mrr_at_50
- value: 61.873
- - type: mrr_at_70
- value: 61.873
+ value: 53.355199222546155
- type: mrr_at_100
- value: 61.873
- - type: mrr_at_200
- value: 61.873
- - type: mrr_at_300
- value: 61.873
- - type: mrr_at_500
- value: 61.873
- - type: mrr_at_700
- value: 61.873
+ value: 53.648197984932665
- type: mrr_at_1000
- value: 61.873
+ value: 53.648197984932665
+ - type: mrr_at_20
+ value: 53.500971817298336
+ - type: mrr_at_3
+ value: 48.63945578231292
+ - type: mrr_at_5
+ value: 51.29251700680272
+ - type: nauc_map_at_1000_diff1
+ value: -8.689968345482955
+ - type: nauc_map_at_1000_max
+ value: -6.9963543014795455
+ - type: nauc_map_at_100_diff1
+ value: -8.727978231600318
+ - type: nauc_map_at_100_max
+ value: -7.952186425971129
+ - type: nauc_map_at_10_diff1
+ value: -2.9526324910232034
+ - type: nauc_map_at_10_max
+ value: -10.068986922862216
+ - type: nauc_map_at_1_diff1
+ value: 11.58796950076508
+ - type: nauc_map_at_1_max
+ value: -13.815104268329259
+ - type: nauc_map_at_20_diff1
+ value: -7.704624587384131
+ - type: nauc_map_at_20_max
+ value: -9.013842231311479
+ - type: nauc_map_at_3_diff1
+ value: 0.3965656398435283
+ - type: nauc_map_at_3_max
+ value: -10.106778815191765
+ - type: nauc_map_at_5_diff1
+ value: -0.47667166254222115
+ - type: nauc_map_at_5_max
+ value: -9.531382459115324
+ - type: nauc_mrr_at_1000_diff1
+ value: 15.425473480157507
+ - type: nauc_mrr_at_1000_max
+ value: -2.1811857816236877
+ - type: nauc_mrr_at_100_diff1
+ value: 15.425473480157507
+ - type: nauc_mrr_at_100_max
+ value: -2.1811857816236877
+ - type: nauc_mrr_at_10_diff1
+ value: 15.903075429674777
+ - type: nauc_mrr_at_10_max
+ value: -2.2906761318729316
+ - type: nauc_mrr_at_1_diff1
+ value: 15.745291718170254
+ - type: nauc_mrr_at_1_max
+ value: 1.3133671227650265
+ - type: nauc_mrr_at_20_diff1
+ value: 15.55498078589214
+ - type: nauc_mrr_at_20_max
+ value: -2.032092110902953
+ - type: nauc_mrr_at_3_diff1
+ value: 12.884821567496274
+ - type: nauc_mrr_at_3_max
+ value: -3.525142776490721
+ - type: nauc_mrr_at_5_diff1
+ value: 16.445959124792818
+ - type: nauc_mrr_at_5_max
+ value: -4.207329992983104
+ - type: nauc_ndcg_at_1000_diff1
+ value: 0.4954475160997072
+ - type: nauc_ndcg_at_1000_max
+ value: -1.4515711306568682
+ - type: nauc_ndcg_at_100_diff1
+ value: -1.6562434532753776
+ - type: nauc_ndcg_at_100_max
+ value: -6.035088636127411
+ - type: nauc_ndcg_at_10_diff1
+ value: 3.1506688231790245
+ - type: nauc_ndcg_at_10_max
+ value: -4.450428581751591
+ - type: nauc_ndcg_at_1_diff1
+ value: 10.902788071421643
+ - type: nauc_ndcg_at_1_max
+ value: -3.5983043940028026
+ - type: nauc_ndcg_at_20_diff1
+ value: -4.129634334760348
+ - type: nauc_ndcg_at_20_max
+ value: -10.377695853633671
+ - type: nauc_ndcg_at_3_diff1
+ value: 3.7390054194490463
+ - type: nauc_ndcg_at_3_max
+ value: -3.6474213357431866
+ - type: nauc_ndcg_at_5_diff1
+ value: 7.729725342433009
+ - type: nauc_ndcg_at_5_max
+ value: -1.9704467553539173
+ - type: nauc_precision_at_1000_diff1
+ value: 2.2605086242588257
+ - type: nauc_precision_at_1000_max
+ value: 29.68572927933118
+ - type: nauc_precision_at_100_diff1
+ value: -3.57573889882659
+ - type: nauc_precision_at_100_max
+ value: 15.778240365801496
+ - type: nauc_precision_at_10_diff1
+ value: 2.426461390379187
+ - type: nauc_precision_at_10_max
+ value: 3.4608738145262774
+ - type: nauc_precision_at_1_diff1
+ value: 15.745291718170254
+ - type: nauc_precision_at_1_max
+ value: 1.3133671227650265
+ - type: nauc_precision_at_20_diff1
+ value: -11.920848565711506
+ - type: nauc_precision_at_20_max
+ value: 2.934435397727988
+ - type: nauc_precision_at_3_diff1
+ value: 3.3909557257855236
+ - type: nauc_precision_at_3_max
+ value: -1.3530477456840486
+ - type: nauc_precision_at_5_diff1
+ value: 12.077987725517026
+ - type: nauc_precision_at_5_max
+ value: 2.598194203393377
+ - type: nauc_recall_at_1000_diff1
+ value: -12.47820280756162
+ - type: nauc_recall_at_1000_max
+ value: 0.18378111872041458
+ - type: nauc_recall_at_100_diff1
+ value: -11.648421171539056
+ - type: nauc_recall_at_100_max
+ value: -8.54128043246034
+ - type: nauc_recall_at_10_diff1
+ value: -4.3154087171431845
+ - type: nauc_recall_at_10_max
+ value: -12.972050661141497
+ - type: nauc_recall_at_1_diff1
+ value: 11.58796950076508
+ - type: nauc_recall_at_1_max
+ value: -13.815104268329259
+ - type: nauc_recall_at_20_diff1
+ value: -14.949511913924157
+ - type: nauc_recall_at_20_max
+ value: -13.18158214788764
+ - type: nauc_recall_at_3_diff1
+ value: -2.997234767804952
+ - type: nauc_recall_at_3_max
+ value: -12.691132477531994
+ - type: nauc_recall_at_5_diff1
+ value: -1.0621644830340549
+ - type: nauc_recall_at_5_max
+ value: -13.702527975951215
+ - type: ndcg_at_1
+ value: 35.714
+ - type: ndcg_at_10
+ value: 28.18
+ - type: ndcg_at_100
+ value: 39.22
+ - type: ndcg_at_1000
+ value: 50.807
+ - type: ndcg_at_20
+ value: 28.979
+ - type: ndcg_at_3
+ value: 31.114000000000004
+ - type: ndcg_at_5
+ value: 29.687
+ - type: precision_at_1
+ value: 36.735
+ - type: precision_at_10
+ value: 24.898
+ - type: precision_at_100
+ value: 7.918
+ - type: precision_at_1000
+ value: 1.5779999999999998
+ - type: precision_at_20
+ value: 18.878
+ - type: precision_at_3
+ value: 31.293
+ - type: precision_at_5
+ value: 29.387999999999998
+ - type: recall_at_1
+ value: 2.806
+ - type: recall_at_10
+ value: 17.776
+ - type: recall_at_100
+ value: 49.41
+ - type: recall_at_1000
+ value: 84.97200000000001
+ - type: recall_at_20
+ value: 26.589000000000002
+ - type: recall_at_3
+ value: 6.866999999999999
+ - type: recall_at_5
+ value: 10.964
- task:
type: Classification
dataset:
@@ -3128,14 +3908,14 @@ model-index:
name: MTEB ToxicConversationsClassification
config: default
split: test
- revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c
+ revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de
metrics:
- type: accuracy
- value: 69.3294
+ value: 91.1376953125
- type: ap
- value: 14.561333393364736
+ value: 40.51219896084815
- type: f1
- value: 53.992309820496466
+ value: 77.5195445434559
- task:
type: Classification
dataset:
@@ -3146,9 +3926,9 @@ model-index:
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a
metrics:
- type: accuracy
- value: 63.63893604980192
+ value: 79.69722693831352
- type: f1
- value: 63.92959380489434
+ value: 80.02969178591319
- task:
type: Clustering
dataset:
@@ -3159,7 +3939,7 @@ model-index:
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449
metrics:
- type: v_measure
- value: 56.270879258659775
+ value: 66.42427742893598
- task:
type: PairClassification
dataset:
@@ -3170,51 +3950,51 @@ model-index:
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1
metrics:
- type: cos_sim_accuracy
- value: 88.71073493473207
+ value: 87.81069321094355
- type: cos_sim_ap
- value: 81.52392540284202
+ value: 78.57014017906349
- type: cos_sim_f1
- value: 74.71162377994676
+ value: 72.38883143743536
- type: cos_sim_precision
- value: 71.89558428885094
+ value: 70.95793208312215
- type: cos_sim_recall
- value: 77.75725593667546
+ value: 73.87862796833772
- type: dot_accuracy
- value: 88.71073493473207
+ value: 87.81069321094355
- type: dot_ap
- value: 81.52394754041109
+ value: 78.5701399541226
- type: dot_f1
- value: 74.71162377994676
+ value: 72.38883143743536
- type: dot_precision
- value: 71.89558428885094
+ value: 70.95793208312215
- type: dot_recall
- value: 77.75725593667546
+ value: 73.87862796833772
- type: euclidean_accuracy
- value: 88.71073493473207
+ value: 87.81069321094355
- type: euclidean_ap
- value: 81.52392035435321
+ value: 78.57015336777854
- type: euclidean_f1
- value: 74.71162377994676
+ value: 72.38883143743536
- type: euclidean_precision
- value: 71.89558428885094
+ value: 70.95793208312215
- type: euclidean_recall
- value: 77.75725593667546
+ value: 73.87862796833772
- type: manhattan_accuracy
- value: 88.47231328604637
+ value: 87.57227156225785
- type: manhattan_ap
- value: 81.22907439267321
+ value: 78.19109731614216
- type: manhattan_f1
- value: 74.3351571446749
+ value: 71.87819856704198
- type: manhattan_precision
- value: 71.78667977390022
+ value: 69.77148534525584
- type: manhattan_recall
- value: 77.0712401055409
+ value: 74.1160949868074
- type: max_accuracy
- value: 88.71073493473207
+ value: 87.81069321094355
- type: max_ap
- value: 81.52394754041109
+ value: 78.57015336777854
- type: max_f1
- value: 74.71162377994676
+ value: 72.38883143743536
- task:
type: PairClassification
dataset:
@@ -3225,68 +4005,83 @@ model-index:
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf
metrics:
- type: cos_sim_accuracy
- value: 89.85136026700819
+ value: 89.95032405790352
- type: cos_sim_ap
- value: 87.7768002924216
+ value: 88.03104739249996
- type: cos_sim_f1
- value: 80.358908624794
+ value: 80.34377190070451
- type: cos_sim_precision
- value: 76.62918209122023
+ value: 77.11534376548892
- type: cos_sim_recall
- value: 84.47028025870034
+ value: 83.85432707114259
- type: dot_accuracy
- value: 89.85136026700819
+ value: 89.95032405790352
- type: dot_ap
- value: 87.77680027889778
+ value: 88.03105328515932
- type: dot_f1
- value: 80.358908624794
+ value: 80.34377190070451
- type: dot_precision
- value: 76.62918209122023
+ value: 77.11534376548892
- type: dot_recall
- value: 84.47028025870034
+ value: 83.85432707114259
- type: euclidean_accuracy
- value: 89.85136026700819
+ value: 89.95032405790352
- type: euclidean_ap
- value: 87.77680174697751
+ value: 88.03105084564575
- type: euclidean_f1
- value: 80.358908624794
+ value: 80.34377190070451
- type: euclidean_precision
- value: 76.62918209122023
+ value: 77.11534376548892
- type: euclidean_recall
- value: 84.47028025870034
+ value: 83.85432707114259
- type: manhattan_accuracy
- value: 89.86300306593705
+ value: 89.88046726433035
- type: manhattan_ap
- value: 87.78613271895861
+ value: 88.01484191858279
- type: manhattan_f1
- value: 80.31831016905645
+ value: 80.34005593993817
- type: manhattan_precision
- value: 76.68230516070304
+ value: 76.95290468133108
- type: manhattan_recall
- value: 84.3162919618109
+ value: 84.03911302740991
- type: max_accuracy
- value: 89.86300306593705
+ value: 89.95032405790352
- type: max_ap
- value: 87.78613271895861
+ value: 88.03105328515932
- type: max_f1
- value: 80.358908624794
+ value: 80.34377190070451
language:
- en
license: cc-by-nc-4.0
---
-Salesforce/SFR-Embedding-Mistral
+Salesforce/SFR-Embedding-2_R
**SFR-Embedding by Salesforce Research.**
-The model is trained on top of [E5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct) and [Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1).
+The model is for **research purposes only**.
+
+More technical details will be updated later. Meanwhile, please refer to our previous work [SFR-Embedding](https://blog.salesforceairesearch.com/sfr-embedded-mistral/) for details.
+
+
+SFR-Embedding Team (∗indicates equal contributors, † indicates co-leaders).
+* Rui Meng*
+* Ye Liu*
+* Shafiq Rayhan Joty
+* Caiming Xiong †
+* Yingbo Zhou †
+* Semih Yavuz †
-This project is for research purposes only. Third-party datasets may be subject to additional terms and conditions under their associated licenses. Please refer to specific papers for more details:
-- [MTEB benchmark](https://arxiv.org/abs/2210.07316)
-- [Mistral](https://arxiv.org/abs/2310.06825)
-- [E5-mistral-7b-instruct](https://arxiv.org/pdf/2401.00368.pdf)
+### Citation
+```bibtex
+@misc{SFR-embedding-2,
+ title={SFR-Embedding-2: Advanced Text Embedding with Multi-stage Training},
+ author={Rui Meng*, Ye Liu*, Shafiq Rayhan Joty, Caiming Xiong, Yingbo Zhou, Semih Yavuz},
+ year={2024},
+ url={https://huggingface.co/Salesforce/SFR-Embedding-2_R}
+}
+```
-More technical details will be updated later.
## How to run
@@ -3324,8 +4119,8 @@ passages = [
]
# load model and tokenizer
-tokenizer = AutoTokenizer.from_pretrained('Salesforce/SFR-Embedding-Mistral')
-model = AutoModel.from_pretrained('Salesforce/SFR-Embedding-Mistral')
+tokenizer = AutoTokenizer.from_pretrained('Salesforce/SFR-Embedding-2_R')
+model = AutoModel.from_pretrained('Salesforce/SFR-Embedding-2_R')
# get the embeddings
max_length = 4096
@@ -3338,7 +4133,7 @@ embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_ma
embeddings = F.normalize(embeddings, p=2, dim=1)
scores = (embeddings[:2] @ embeddings[2:].T) * 100
print(scores.tolist())
-# [[86.7153549194336, 36.64569091796875], [35.00493621826172, 82.0738525390625]]
+# [[40.132083892822266, 25.032529830932617], [15.006855010986328, 39.93733215332031]]
```
### Sentence Transformers
@@ -3346,7 +4141,7 @@ print(scores.tolist())
from sentence_transformers import SentenceTransformer, util
-model = SentenceTransformer("Salesforce/SFR-Embedding-Mistral")
+model = SentenceTransformer("Salesforce/SFR-Embedding-2_R")
def get_detailed_instruct(task_description: str, query: str) -> str:
return f'Instruct: {task_description}\nQuery: {query}'
@@ -3366,30 +4161,7 @@ passages = [
embeddings = model.encode(queries + passages)
scores = util.cos_sim(embeddings[:2], embeddings[2:]) * 100
print(scores.tolist())
-# [[86.71537780761719, 36.645721435546875], [35.00497055053711, 82.07388305664062]]
-```
-
-### MTEB Benchmark Evaluation
-Check out [unilm/e5](https://github.com/microsoft/unilm/tree/master/e5) to reproduce evaluation results on the [BEIR](https://arxiv.org/abs/2104.08663) and [MTEB](https://arxiv.org/abs/2210.07316) benchmark.
-
-
-SFR-Embedding Team (∗indicates lead contributors).
-* Rui Meng*
-* Ye Liu*
-* Shafiq Rayhan Joty
-* Caiming Xiong
-* Yingbo Zhou
-* Semih Yavuz
-
-### Citation
-```bibtex
-@misc{SFRAIResearch2024,
- title={SFR-Embedding-Mistral:Enhance Text Retrieval with Transfer Learning},
- author={Rui Meng, Ye Liu, Shafiq Rayhan Joty, Caiming Xiong, Yingbo Zhou, Semih Yavuz},
- howpublished={Salesforce AI Research Blog},
- year={2024},
- url={https://blog.salesforceairesearch.com/sfr-embedded-mistral/}
-}
+# [[40.132083892822266, 25.032529830932617], [15.006855010986328, 39.93733215332031]]
```